commit 86db9aae8e6c3c3331e002262a84445fcd569258 Author: wehub-resource-sync Date: Mon Jul 13 13:34:55 2026 +0800 chore: import upstream snapshot with attribution diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..d7b506d --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,86 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# Sample workflow for building and deploying a Jekyll site to GitHub Pages +name: Documentation + +on: + push: + branches: ["main"] + paths: ["docs/**"] # only changes in the docs directory triger the workflow + + pull_request: + branches: ["main"] + types: ["closed"] + paths: ["docs/**"] + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + # Build job + build: + runs-on: ubuntu-latest + defaults: + run: + working-directory: docs + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3' # Not needed with a .ruby-version file + bundler-cache: true # runs 'bundle install' and caches installed gems automatically + cache-version: 0 # Increment this number if you need to re-download cached gems + working-directory: '${{ github.workspace }}/docs' + + - name: Setup Pages + id: pages + uses: actions/configure-pages@v3 + + - name: Build with Jekyll + # Outputs to the './_site' directory by default + run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}" + env: + JEKYLL_ENV: production + + - name: Upload artifact + # Automatically uploads an artifact from the './_site' directory by default + uses: actions/upload-pages-artifact@v3 + with: + path: "docs/_site/" + + + # Deployment job + deploy: + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b96c201 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# Backup fiels +~* + +# Python build / packaging +build/ +dist/ +__pycache__/ +llmware.egg-info/ +venv/ + +# Jekyll +docs/_site/ + + +# Mac +.DS_Store/ + +# Testing +.env \ No newline at end of file diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..92536a9 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.0 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..93d5acf --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2023-2026 by AI Bloks for LLMWare + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..fd78377 --- /dev/null +++ b/NOTICE @@ -0,0 +1,670 @@ + +Copyright 2023-2026 by llmware + +NOTICE - Overview + +LLMWare Models. This software contains links to the LLMWare public model repository currently hosted on Huggingface (www.huggingface.co/llmware). LLMWare models in this repository are generally licensed under a separate Apache License 2.0 license, e.g., https://www.apache.org/licenses/LICENSE-2.0. Where there are any exceptions to Apache 2.0, it is noted on the Model Card, e.g., Llama 2 Community License or CC-BY-SA-4.0, and is following licensing conditions of the underlying base models. All of the LLMWare models that are listed in the ModelCatalog in this repository are permissively licensed and may be used for commercial purposes in conjunction with llmware, subject to the license terms found with the model cards. + +3rd Party Models. The llmware software package also provides the ability to use other 3rd party models in conjunction with the llmware software. Use of any third party models is subject to the licensing terms of those models. + +Sample Testing Files. This software includes sample files that are made available for testing, including documents and audio files. These files have either been derived from open public domain sources, or produced as original materials by the llmware team to be used as test samples and examples. These files are made available solely for purposes of testing and illustrating the use of key llmware functions, and should not be used, distributed or published in any other manner. The audio files, in particular, may be subject to copyright protections that preclude any other form of publishing beyond limited 'fair use' as testing files. + +Compiled Enabling Components. This software consists primarily of Python source code, but also includes back-end enabling components consisting of prebuilt compiled C/C++ shared libraries and supporting 3rd party dependencies: + + * libpdf_llmware - this is a PDF parser written in C developed and copyrighted separately by llmware, which implements the ISO-32000 specification. The compiled parser is provided as a shared library as part of the llmware software package, under an Apache 2.0 license, but the source code has not been provided in the llmware project currently. The interfaces to the shared library are in the Python source code in the Parser module, which enable the user to access, configure and control the behavior of the PDF parser as part of building applications and other solutions with llmware. + + * liboffice_llmware - this is an Office XML parser written in C developed and copyrighted by llmware, which implements the ISO-29500 Office Open XML specification for documents that conform with Word, Powerpoint and Excel formats. The compiled parser is provided as a shared library as part of the llmware software package, under an Apache 2.0 license, but the source code has not been provided in the llmware project currently. The interfaces to the shared library are in the Python source code in the Parser module, which enable the user to access, configure and control the behavior of the Office parser as part of building applications and other solutions with llmware. + + * libgraph_llmware - this module provides a set of NLP-based utility functions, written in C developed and copyrighted by llmware, and exposed in the Python code in the graph module. The compiled code is provided as a shared library as part of the llmware software package, under an Apache 2.0 license, but the source code has not been released in the llmware project currently. + + * llama.cpp / GGUF - this a prebuilt shared library implementation of Llama.CPP and associated GGML middleware components, licensed under a MIT license with the source code and all other information in the repository: www.github.com/ggerganov/llama.cpp.git. The llmware implementation generally follows the core Python interface for Llama.CPP as provided in the repository: www.github.com/abetlen/llama-cpp-python.git, although there will be differences from time-to-time, and llmware may have additional features not found in other Python interfaces and vice versa. The llmware shared library implementation is regularly updated and re-compiled, and tested for integration into llmware, but may depart from the current source code of llama.cpp. + + * whisper.cpp / GGML - this is a prebuilt shared library implementation of Whisper.CPP and associated GGML middleware components, licensed under a MIT license with the source code and all other information in the repository: www.github.com/ggerganov/whisper.cpp.git. The llmware Python interface for Whisper.CPP was developed originally, but took inspiration from www.github.com/carloscdias/whisper-cpp-python.git, which is provided under a MIT license. The llmware shared library implementation is regularly updated and recompiled, and tested for integration into llmware, but may depart from the current source code of whisper.cpp. + + Why shared libraries rather than source code? The objective of including these prebuilt shared libraries in this project is to enable a complete LLM-based pipeline development framework that works "out of the box" on multiple platforms, and also enables a developer to rapidly and intuitively scale to large, complex solutions. We view these shared libraries as enabling components with the core of the llmware project consisting of the higher-level classes and functions exposed in the Python source code. In the future, we may revisit the decision to publish the source code for the llmware parsers (e.g., libpdf_llmware, liboffice_llmware, libgraph_llmware), and/or provide under a separate project repository focused on lower-level components and in C/C++. Generally, these parsers are complex, standalone, low-level code that implements the arcane rules of various ISO standards, and for the foreseeable future, we do not have the bandwidth to manage a full open source documentation, testing, and lifecycle around that source code base - without it overwhelming the main objective of llmware. + +Vector Databases. This software provides integration to a wide variety of vector database resources, using the open source Python drivers and associated SDKs provided by the vector database vendor with the licensing details outlined below. Users need to install any vector databases separately and independently, subject to the licensing terms from those vendors. llmware does not provide any licenses to vector databases, and llmware does not require the use of a vector database for a wide range of use cases. In several Fast Start examples, we demonstrate how to use "no-install" vector databases, such as FAISS (MIT license), ChromaDB (Apache 2.0 license - www.github.com/chroma-core/chroma.git) and LanceDB (Apache 2.0 license - www.github.com/lancedb/lancedb.git), but no license is provided to those separate resources. + +General Purpose Databases. This software provides integration to three core database resources - Postgres, MongoDB and SQLite. llmware connects to these resources using open source Python and C drivers. Users need to install these databases separately and indendently, subject to the licensing terms from those vendors. llmware does not provide any licenses to databases, and llmware does not require the use of a database for a wide range of use cases. + +Huggingface Integration. This software provides features and functions that enable access to models, tokenizers and datasets hosted in Huggingface repositories and accessible via the Huggingface transformers, tokenizers, datasets and huggingface-hub libraries. In providing interfaces to these resources, llmware uses transformers, in particular, which is provided by Huggingface under an Apache 2.0 license (see www.github.com/huggingface/transformers.git). In addition, related to these interfaces, llmware provides code that was influenced, derived, and in a few limited cases, copied, from transformers source code to facilitate the integration. Some of this transformers code, especially the underlying model class code, is subject to additional copyright notices from HuggingFace, Google AI, EleutherAI, NVIDIA and potentially others for specific models. For use of any individual third party model, accessed through a Huggingface repository, we recommend reviewing the individual model card to confirm the licensing terms and other associated copyrights. + +================================================================================================= + +Open Source Dependencies and Optional Components + +Please note that the list below includes 'first-level' dependencies but does not include potential 'second-level' dependencies included within the components outlined below. + +3rd Party Open Source libraries, drivers and tools in Python (pip install) and C/C++ dependencies used in conjunction for the llmware software package: + +3-clause BSD License (https://opensource.org/license/bsd-3-clause/) + * Software: libzip (https://libzip.org/) [C library] + * Software: numpy (https://github.com/numpy/numpy) [Python pip install] + * Software: torch (https://github.com/pytorch/pytorch) [Python pip install] + * Software: colorama (https://github.com/tartley/colorama) [Python pip install] + * Software - Optional - lxml (https://github.com/lxml/lxml) [Optional / Python pip install] + +Apache License 2.0 (https://www.apache.org/licenses/LICENSE-2.0) + * Software: boto3 (https://github.com/boto/boto3) [Python pip install] + * Software: mongo-c-driver (https://github.com/mongodb/mongo-c-driver) [C library] + * Software: pymongo (http://github.com/mongodb/mongo-python-driver) [Python pip install] + * Software: tokenizers (https://github.com/huggingface/tokenizers) [Python pip install] + * Software: transformers (https://github.com/huggingface/transformers) [Python pip install] + * Software: huggingface-hub (https://github.com/huggingface/huggingface-hub) [Python pip install] + * Software: requests (https://github.com/psf/requests) [Python pip install] + * Software - Optional - pymilvus (https://github.com/milvus-io/pymilvus) [Optional / Python pip install] + * Software - Optional - pytesseract (https://github.com/madmaze/pytesseract) [Optional / Python pip install] + * Software - Optional - datasets (https://github.com/huggingface/datasets) [Optional / Python pip install] + * Software - Optional - sentence-transformers (https://github.com/UKPLabs/sentence-transformers) [Optional / Python pip install] + * Software - Optional - yfinance (https://github.com/ranaroussi/yfinance) [Optional / Python pip install] + * Software - Optional - chromadb (https://github.com/chroma-core/chroma) [Optional / Python pip install] + * Optional - Optional - neo4j-python-driver (https://github.com/neo4j/neo4j-python-driver) [Optional / Python pip install] + * Optional - Optional - Tiny-Llama base model (https://github.com/jzhang38/TinyLlama) [Separate Download and install] + * Optional - Optional - tesseract (https://github.com/tesseract-ocr/tesseract) [Optional / not included / C library] + +ICS License (https://opensource.org/license/isc-license-txt) + * Software: librosa - source code and license at: https://github.com/librosa/librosa [Python pip install] + + +Database Drivers used in llmware + * Mongo C driver - libmongoc / libbson - Apache 2.0 (https://www.github.com/mongodb/mongo-c-driver) [C library] + * PostgreSQL C driver - libpq - PostgresSQL Global Development Group (https://www.github.com/postgres/postgres/blob/master/copyright) [C library] + * PostgreSQL psycopg - unmodified and linked dynamically from pypi release - GNU Lesser General Public License (https://www.github.com/psyocopg/psycopg/LICENSE.txt) [Python pip install] + * SQLite C driver - libsqlite3 - (https://www.github.com/LuaDist/libsqlite3) [C library] + * psycopg2 & psycopg2-binary - GNU Lesser General Public License - no modifications - it is provided through pip install as standard python library + + +PNG Reference Library License + * libpng: source code and license (https://github.com/pnggroup/libpng) [C library] + +libtiff License (https://spdx.org/licenses/libtiff.html) + * Software: libtiff (http://www.libtiff.org/) [C library] + +MIT License (https://opensource.org/license/mit/) + * Software: libxml2 (https://github/com/GNOME/libxml2) [C library] + * Software: llama.cpp (https://github.com/ggerganov/llama.cpp) [C/C++ library] + * Software: whisper.cpp (https://github.com/ggerganov/whisper.cpp) [C/C++ library] + * Software: openai (https://github.com/openai/openai-python) [Python pip install] + * Software: Wikipedia-API (https://github.com/martin-majlis/Wikipedia-API) [Python pip install] + * Software: einops (https://github.com/arogozhinov/einops) [Python pip install] + * Software - Optional - beautifulsoup4 (https://pypi.org/project/beautifulsoup4/) [Optional / Python pip install] + * Software - Optional - Whisper models - openai (https://github.com/openai/whisper) [Separate download and install] + * Software - Optional - anthropic (https://github.com/anthropics/anthropic-sdk-python) [Optional / Python pip install] + * Software - Optional - faise-cpu (https://github.com/kyamagu/faiss-wheels) [Optional / Python pip install] + * Software - Optional - cohere (https://github.com/cohere-ai/cohere-python) [Optional / Python pip install] + * Software - Optional - tabulate (https://github.com/astanin/python-tabulate) [Optional / Python pip install] + * Software - Optional - pdf2image (https://github.com/Belval/pdf2image) [Optional / Python pip install] + * Software - Optional - word2number (https://github.com/akshaynagpal/w2n) [Optional / Python pip install] + +zlib License (https://github.com/madler/zlib/blob/develop/LICENSE) + * Software: zlib (https://www.zlib.net/) [C library] + +Databases - separate and optional components that may be used in conjunction with llmware and require user to license and install independently: + + * Postgres: source code for Postgres database provided at: https://github.com/postgres. Copyright is by PostgreSQL Global Development Group and Regents of the University of California - (https://github.com/postgres/postgres/blob/master/copyright) - with license terms outlined below in the Copyright Notices section. [Separate download and install] + + * MongoDB: source code for Community edition provided at: https://github.com/mongodb/mongo - "MongoDB is free and the source is available" subject to the Server Side Public License (SSPL) v1 - (https://github.com/mongodb/mongo/blob/master/LICENSE-Community.txt). [Separate download and install] + + * SQLite: source code provided at: https://github.com/sqlite/sqlite - not subject to copyright - "May you share freely, never taking more than you give." [Separate download and install] + +Vector Databases - separate and optional open source components that may be used in conjunction with llmware and require user to license and install independently: + + * Milvus: Apache 2.0 license - source code and license at: https://github.com/milvus-io/milvus + * Qdrant: Apache 2.0 license - source code and license at: https://github.com/qdrant/qdrant + * ChromaDB: Apache 2.0 license - source code and license at: https://github.com/chroma-core/chroma + * PGVector: Postgres Development Group license - source code and license at: https://github.com/pgvector/pgvector + * Neo4J: GPL-3.0 license - source and license at: https://github.com/neo4j/neo4j + * FAISS: MIT license - source code and license at: https://github.com/facebookresearch/faiss + * LanceDB: Apache 2.0 license - source code and license at: www.github.com/lancedb/lancedb.git + +================================================================================================= + +Required Copyright Notices Per Open Source Licenses for Components Distributed Directly with a Full Clone of the Repository - does not include components installed through a separate pip install process, or additional components used in conjunction with LLMWare, but installed independently from this repository. + +Name: llama.cpp +License: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Name: whisper.cpp +License: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Name: libxml2 +License: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Name: einops +License: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Name: wikipedia-api +License: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Name: openai (python sdk) +License: MIT + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + +Name: mongo-c-driver (libmongoc and libbson) +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: pymongo +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: boto3 +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: requests +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: transformers (Huggingface) +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: tokenizers (Huggingface) +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: huggingface_hub (HuggingFace) +License: Apache 2.0 Copyright Notice + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Name: librosa +License: ISC License +Copyright (c) 2013--2023, librosa development team. +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +Name: psycopg [no modifications - it is provided through pip install as standard python library] +License: Lesser GPL + +psycopg2 is free software: you can redistribute it and/or modify it +under the terms of the GNU Lesser General Public License as published +by the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +psycopg2 is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +License for more details. + +In addition, as a special exception, the copyright holders give +permission to link this program with the OpenSSL library (or with +modified versions of OpenSSL that use the same license as OpenSSL), +and distribute linked combinations including the two. + +You must obey the GNU Lesser General Public License in all respects for +all of the code used other than OpenSSL. If you modify file(s) with this +exception, you may extend this exception to your version of the file(s), +but you are not obligated to do so. If you do not wish to do so, delete +this exception statement from your version. If you delete this exception +statement from all source files in the program, then also delete it here. + +You should have received a copy of the GNU Lesser General Public License +along with psycopg2 (see the doc/ directory.) +If not, see . + + + +Name: numpy +License: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: torch +License: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: colorama +License: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: libzip +License: BSD-3-Clause + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: pgvector +License: Postgres License + +Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + +Portions Copyright (c) 1994, The Regents of the University of California + +Permission to use, copy, modify, and distribute this software and its +documentation for any purpose, without fee, and without a written agreement +is hereby granted, provided that the above copyright notice and this +paragraph and the following two paragraphs appear in all copies. + +IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR +DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING +LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS +DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS +ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO +PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + + +Name: libpq +License: GNU LESSER GENERAL PUBLIC LICENSE & Postgres License + +PostgreSQL License + +PostgreSQL is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses. + +PostgreSQL Database Management System +(formerly known as Postgres, then as Postgres95) + +Portions Copyright © 1996-2024, The PostgreSQL Global Development Group + +Portions Copyright © 1994, The Regents of the University of California + +Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. + +IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + + +GNU Lesser General Public License - Version 3, 29 June 2007 + +Copyright © 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + +This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. + +0. Additional Definitions. +As used herein, “this License” refers to version 3 of the GNU Lesser General Public License, and the “GNU GPL” refers to version 3 of the GNU General Public License. + +“The Library” refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. + +An “Application” is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. + +A “Combined Work” is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the “Linked Version”. + +The “Minimal Corresponding Source” for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. + +The “Corresponding Application Code” for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. + +1. Exception to Section 3 of the GNU GPL. +You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. + +2. Conveying Modified Versions. +If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: + +a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or +b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. +3. Object Code Incorporating Material from Library Header Files. +The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: + +a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. +b) Accompany the object code with a copy of the GNU GPL and this license document. +4. Combined Works. +You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: + +a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. +b) Accompany the Combined Work with a copy of the GNU GPL and this license document. +c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. +d) Do one of the following: +0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. +1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. +e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) +5. Combined Libraries. +You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: + +a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. +b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. +6. Revised Versions of the GNU Lesser General Public License. +The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. + +If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. + + +Name: libpng +License: PNG Reference Library Copyright Noticee and License version 2 + + * Copyright (c) 1995-2024 The PNG Reference Library Authors. + * Copyright (c) 2018-2024 Cosmin Truta. + * Copyright (c) 2000-2002, 2004, 2006-2018 Glenn Randers-Pehrson. + * Copyright (c) 1996-1997 Andreas Dilger. + * Copyright (c) 1995-1996 Guy Eric Schalnat, Group 42, Inc. + +The software is supplied "as is", without warranty of any kind, +express or implied, including, without limitation, the warranties +of merchantability, fitness for a particular purpose, title, and +non-infringement. In no event shall the Copyright owners, or +anyone distributing the software, be liable for any damages or +other liability, whether in contract, tort or otherwise, arising +from, out of, or in connection with the software, or the use or +other dealings in the software, even if advised of the possibility +of such damage. + +Permission is hereby granted to use, copy, modify, and distribute +this software, or portions hereof, for any purpose, without fee, +subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you + must not claim that you wrote the original software. If you + use this software in a product, an acknowledgment in the product + documentation would be appreciated, but is not required. + + 2. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + + 3. This Copyright notice may not be removed or altered from any + source or altered source distribution. + +Name: libtiff +License: Custom (https://github.com/libsdl-org/libtiff) + +Silicon Graphics has seen fit to allow us to give this work away. It is free. There is no support or guarantee of any sort as to its operations, correctness, or whatever. If you do anything useful with all or parts of it you need to honor the copyright notices. I would also be interested in knowing about it and, hopefully, be acknowledged. + +The legal way of saying that is: + +Copyright (c) 1988-1997 Sam Leffler Copyright (c) 1991-1997 Silicon Graphics, Inc. + +Permission to use, copy, modify, distribute, and sell this software and its documentation for any purpose is hereby granted without fee, provided that (i) the above copyright notices and this permission notice appear in all copies of the software and related documentation, and (ii) the names of Sam Leffler and Silicon Graphics may not be used in any advertising or publicity relating to the software without the specific, prior written permission of Sam Leffler and Silicon Graphics. + +THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND, EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +Name: zlib +License: Custom (https://github.com/madler/zlib?tab=License-1-ov-file) + +Copyright notice: + + (C) 1995-2024 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + +================================================================================================= + +Citations for Open Source Software, Models and Research used in the development of llmware: + +Transformers - https://github.com/huggingface/transformers + +@inproceedings{wolf-etal-2020-transformers, + title = "Transformers: State-of-the-Art Natural Language Processing", + author = "Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush", + booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing: System Demonstrations", + month = oct, + year = "2020", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://www.aclweb.org/anthology/2020.emnlp-demos.6", + pages = "38--45" +} + +Sentence Transformers - https://github.com/UKPLab/sentence-transformers + +@inproceedings{reimers-2019-sentence-bert, + title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks", + author = "Reimers, Nils and Gurevych, Iryna", + booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing", + month = "11", + year = "2019", + publisher = "Association for Computational Linguistics", + url = "https://arxiv.org/abs/1908.10084", +} + +Pytorch - https://github.com/pytorch/blob/main/CITATION.cff (with full and updated contributor list) + + title: PyTorch + authors: PyTorch Team + url: https://pytorch.org + type: conference-paper + title: "PyTorch 2: Faster Machine Learning Through Dynamic Python Bytecode Transformation and Graph Compilation" + collection-title: "29th ACM International Conference on Architectural Support for Programming Languages and Operating Systems, Volume 2 (ASPLOS '24)" + collection-type: proceedings + month: 4 + year: 2024 + publisher: + name: ACM + url: "https://pytorch.org/assets/pytorch2-2.pdf" + + Paszke, A., Gross, S., Massa, F., Lerer, A., Bradbury, J., Chanan, G., Killeen, T., Lin, Z., Gimelshein, N., Antiga, L., Desmaison, A., Kopf, A., Yang, E., DeVito, Z., Raison, M., Tejani, A., Chilamkurthy, S., Steiner, B., Fang, L., Bai, J., & Chintala, S. (2019). PyTorch: An Imperative Style, High-Performance Deep Learning Library [Conference paper]. Advances in Neural Information Processing Systems 32, 8024–8035. http://papers.neurips.cc/paper/9015-pytorch-an-imperative-style-high-performance-deep-learning-library.pdf + + +Whisper - OpenAI - https://github.com/openai/whisper & www.huggingface.co/openai/whisper + +@misc{radford2022whisper, + doi = {10.48550/ARXIV.2212.04356}, + url = {https://arxiv.org/abs/2212.04356}, + author = {Radford, Alec and Kim, Jong Wook and Xu, Tao and Brockman, Greg and McLeavey, Christine and Sutskever, Ilya}, + title = {Robust Speech Recognition via Large-Scale Weak Supervision}, + publisher = {arXiv}, + year = {2022}, + copyright = {arXiv.org perpetual, non-exclusive license} +} + +BERT +Turc, Iulia; Chang, Ming-Wei; Lee, Kenton; Toutanova, Kristina. Well-Read Students Learn Better: On the Importance of Pre-training Compact Models, arXiv preprint arXiv:1908.08962v2 (2019). + +Devlin, Jacob; Chang, Ming-Wei; Lee, Kenton; Toutanova, Kristina. BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding, arXiv preprint arXiv:1810.04805 (2018). + +GPT2 +Radford, Alec; Wu, Jeff; Child, Rewon; Luan, David; Amodei, Dario; Sutskever, Ilya. Language Models are Unsupervised Multitask Learners. (2019) + +Tiny Llama +@misc{zhang2024tinyllama, + title={TinyLlama: An Open-Source Small Language Model}, + author={Peiyuan Zhang and Guangtao Zeng and Tianduo Wang and Wei Lu}, + year={2024}, + eprint={2401.02385}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} + +StableLM-3b-4e1t Model (StabilityAI) +@misc{StableLM-3B-4E1T, + url={[https://huggingface.co/stabilityai/stablelm-3b-4e1t](https://huggingface.co/stabilityai/stablelm-3b-4e1t)}, + title={StableLM 3B 4E1T}, + author={Tow, Jonathan and Bellagente, Marco and Mahan, Dakota and Riquelme, Carlos} +} + +Sheared Llama +@article{xia2023sheared, title={Sheared LLaMA: Accelerating Language Model Pre-training via Structured Pruning}, author={Xia, Mengzhou and Gao, Tianyu, and Zeng Zhiyuan, and Chen Danqi}, year={2023} } + + +FAISS +@article{douze2024faiss, + title={The Faiss library}, + author={Matthijs Douze and Alexandr Guzhva and Chengqi Deng and Jeff Johnson and Gergely Szilvasy and Pierre-Emmanuel Mazaré and Maria Lomeli and Lucas Hosseini and Hervé Jégou}, + year={2024}, + eprint={2401.08281}, + archivePrefix={arXiv}, + primaryClass={cs.LG} +} + + diff --git a/README.md b/README.md new file mode 100644 index 0000000..6b43471 --- /dev/null +++ b/README.md @@ -0,0 +1,1123 @@ +# llmware +![Static Badge](https://img.shields.io/badge/python-3.10_%7C_3.11%7C_3.12%7C_3.13%7C_3.14-blue?color=blue) +![PyPI - Version](https://img.shields.io/pypi/v/llmware?color=blue) +[![members](https://discord-live-members-count-badge.vercel.app/api/discord-members?guildId=1179245642770559067&label=discord%20members&color=5865F2)](https://discord.gg/bphreFK4NJ) +[![Documentation](https://github.com/llmware-ai/llmware/actions/workflows/pages.yml/badge.svg)](https://github.com/llmware-ai/llmware/actions/workflows/pages.yml) + +## 🧰🛠️ Unified framework for building knowledge-based local, private, secure LLM-based applications + +`llmware` is optimized for AI PC and local laptop, edge and self-hosted deployment across a wide range of Windows, Mac and Linux platforms, with support for GGUF, OpenVINO, ONNXRuntime, ONNXRuntime-QNN (Qualcomm), WindowsLocalFoundry, and Pytorch, providing a high-level interface that makes it easy to leverage the right inferencing technology optimized for the target platform. + + `llmware` has two main components: + + 1. **Model catalog with 300+ models** - models prepackaged in quantized, optimized formats, to leverage on device GPU and NPU capabilities, with support for major open source model families and 50+ llmware finetuned SLIM, Bling, Dragon and Industry-Bert models specialized for key tasks in enterprise process automation. Also supports leading cloud models from OpenAI, Anthropic and Google. + + 2. **RAG Pipeline** - integrated components for the full lifecycle of connecting knowledge sources to generative AI models with wide range of document parsing and ingestion capabilities, and the ability to create scalable knowledge bases. + +By bringing together both of these components, `llmware` offers a comprehensive set of tools to rapidly build knowledge-based enterprise LLM applications. + +Our vision is that AI should be sustainable, accurate, and cost-effective, using the smallest possible compute footprint to get the job done. + +Virtually all of our examples and models can be run on device - get started right away on your laptop. + +[Join us on Discord](https://discord.gg/MhZn5Nc39h) | [Watch Youtube Tutorials](https://www.youtube.com/@llmware) | [Explore our Model Families on Huggingface](https://www.huggingface.co/llmware) + + +## 🎯 Key features +Writing code with`llmware` is based on a few main concepts: + +
+Model Catalog: Access all models the same way with easy lookup, regardless of underlying implementation. + + + +```python +# 300+ Models in Catalog with 50+ RAG-optimized BLING, DRAGON and Industry BERT models +# Full support for GGUF, OpenVINO, Onnxruntime, HuggingFace, Sentence Transformers and major API-based models +# Easy to extend to add custom models - see examples + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# all models accessed through the ModelCatalog +models = ModelCatalog().list_all_models() + +# to use any model in the ModelCatalog - "load_model" method and pass the model_name parameter +my_model = ModelCatalog().load_model("llmware/bling-phi-3-gguf") + +# call model with: inference +output = my_model.inference("what is the future of AI?", add_context="Here is the article to read") + +# call model with: stream +for token in my_model.stream("What is the future of AI?"): + print(token, end="") + +# to integrate model into a Prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") +response = prompter.prompt_main("what is the future of AI?", context="Insert Sources of information") +``` + +
+ +
+Library: ingest, organize and index a collection of knowledge at scale - Parse, Text Chunk and Embed. + +```python + +from llmware.library import Library + +# to parse and text chunk a set of documents (pdf, pptx, docx, xlsx, txt, csv, md, json/jsonl, wav, png, jpg, html) + +# step 1 - create a library, which is the 'knowledge-base container' construct +# - libraries have both text collection (DB) resources, and file resources (e.g., llmware_data/accounts/{library_name}) +# - embeddings and queries are run against a library + +lib = Library().create_new_library("my_library") + +# step 2 - add_files is the universal ingestion function - point it at a local file folder with mixed file types +# - files will be routed by file extension to the correct parser, parsed, text chunked and indexed in text collection DB + +lib.add_files("/folder/path/to/my/files") + +# to install an embedding on a library - pick an embedding model and vector_db +lib.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="milvus", batch_size=500) + +# to add a second embedding to the same library (mix-and-match models + vector db) +lib.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="chromadb", batch_size=100) + +# easy to create multiple libraries for different projects and groups + +finance_lib = Library().create_new_library("finance_q4_2023") +finance_lib.add_files("/finance_folder/") + +hr_lib = Library().create_new_library("hr_policies") +hr_lib.add_files("/hr_folder/") + +# pull library card with key metadata - documents, text chunks, images, tables, embedding record +lib_card = Library().get_library_card("my_library") + +# see all libraries +all_my_libs = Library().get_all_library_cards() + +``` +
+ +
+Query: query libraries with mix of text, semantic, hybrid, metadata, and custom filters. + +```python + +from llmware.retrieval import Query +from llmware.library import Library + +# step 1 - load the previously created library +lib = Library().load_library("my_library") + +# step 2 - create a query object and pass the library +q = Query(lib) + +# step 3 - run lots of different queries (many other options in the examples) + +# basic text query +results1 = q.text_query("text query", result_count=20, exact_mode=False) + +# semantic query +results2 = q.semantic_query("semantic query", result_count=10) + +# combining a text query restricted to only certain documents in the library and "exact" match to the query +results3 = q.text_query_with_document_filter("new query", {"file_name": "selected file name"}, exact_mode=True) + +# to apply a specific embedding (if multiple on library), pass the names when creating the query object +q2 = Query(lib, embedding_model_name="mini_lm_sbert", vector_db="milvus") +results4 = q2.semantic_query("new semantic query") +``` + +
+ +
+Prompt with Sources: the easiest way to combine knowledge retrieval with a LLM inference. + +```python + +from llmware.prompts import Prompt +from llmware.retrieval import Query +from llmware.library import Library + +# build a prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") + +# add a file -> file is parsed, text chunked, filtered by query, and then packaged as model-ready context, +# including in batches, if needed, to fit the model context window + +source = prompter.add_source_document("/folder/to/one/doc/", "filename", query="fast query") + +# attach query results (from a Query) into a Prompt +my_lib = Library().load_library("my_library") +results = Query(my_lib).query("my query") +source2 = prompter.add_source_query_results(results) + +# run a new query against a library and load directly into a prompt +source3 = prompter.add_source_new_query(my_lib, query="my new query", query_type="semantic", result_count=15) + +# to run inference with 'prompt with sources' +responses = prompter.prompt_with_source("my query") + +# to run fact-checks - post inference +fact_check = prompter.evidence_check_sources(responses) + +# to view source materials (batched 'model-ready' and attached to prompt) +source_materials = prompter.review_sources_summary() + +# to see the full prompt history +prompt_history = prompter.get_current_history() +``` + +
+ +
+RAG-Optimized Models - 1-7B parameter models designed for RAG workflow integration and running locally. + +``` +""" This 'Hello World' example demonstrates how to get started using local BLING models with provided context, using both +Pytorch and GGUF versions. """ + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +# this is the main script to be run + +def bling_meets_llmware_hello_world (model_name): + + t0 = time.time() + + # load the questions + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + # load the model + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"] + , prompt_name="default_with_context",temperature=0.30) + + # print out the results + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # list of 'rag-instruct' laptop-ready small bling models on HuggingFace + + pytorch_models = ["llmware/bling-1b-0.1", # most popular + "llmware/bling-tiny-llama-v0", # fastest + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3" # most accurate (and newest) + ] + + # Quantized GGUF versions generally load faster and run nicely on a laptop with at least 16 GB of RAM + gguf_models = ["bling-phi-3-gguf", "bling-stablelm-3b-tool", "dragon-llama-answer-tool", "dragon-yi-answer-tool", "dragon-mistral-answer-tool"] + + # try model from either pytorch or gguf model list + # the newest (and most accurate) is 'bling-phi-3-gguf' + + bling_meets_llmware_hello_world(gguf_models[0] + + # check out the model card on Huggingface for RAG benchmark test performance results and other useful information +``` + +
+ +
+Simple-to-Scale Database Options - integrated data stores from laptop to parallelized cluster. + +```python + +from llmware.configs import LLMWareConfig + +# to set the collection database - mongo, sqlite, postgres +LLMWareConfig().set_active_db("mongo") + +# to set the vector database (or declare when installing) +# --options: milvus, pg_vector (postgres), redis, qdrant, faiss, pinecone, mongo atlas +LLMWareConfig().set_vector_db("milvus") + +# for fast start - no installations required +LLMWareConfig().set_active_db("sqlite") +LLMWareConfig().set_vector_db("chromadb") # try also faiss and lancedb + +# for single postgres deployment +LLMWareConfig().set_active_db("postgres") +LLMWareConfig().set_vector_db("postgres") + +# to install mongo, milvus, postgres - see the docker-compose scripts as well as examples + +``` + +
+ +
+ + Agents with Function Calls and SLIM Models + +```python + +from llmware.agents import LLMfx + +text = ("Tesla stock fell 8% in premarket trading after reporting fourth-quarter revenue and profit that " + "missed analysts’ estimates. The electric vehicle company also warned that vehicle volume growth in " + "2024 'may be notably lower' than last year’s growth rate. Automotive revenue, meanwhile, increased " + "just 1% from a year earlier, partly because the EVs were selling for less than they had in the past. " + "Tesla implemented steep price cuts in the second half of the year around the world. In a Wednesday " + "presentation, the company warned investors that it’s 'currently between two major growth waves.'") + +# create an agent using LLMfx class +agent = LLMfx() + +# load text to process +agent.load_work(text) + +# load 'models' as 'tools' to be used in analysis process +agent.load_tool("sentiment") +agent.load_tool("extract") +agent.load_tool("topics") +agent.load_tool("boolean") + +# run function calls using different tools +agent.sentiment() +agent.topics() +agent.extract(params=["company"]) +agent.extract(params=["automotive revenue growth"]) +agent.xsum() +agent.boolean(params=["is 2024 growth expected to be strong? (explain)"]) + +# at end of processing, show the report that was automatically aggregated by key +report = agent.show_report() + +# displays a summary of the activity in the process +activity_summary = agent.activity_summary() + +# list of the responses gathered +for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + +output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + +``` + +
+
+ + 🚀 Start coding - Quick Start for RAG + +```python +# This example illustrates a simple contract analysis +# using a RAG-optimized LLM running locally + +import os +import re +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + +def contract_analysis_on_laptop (model_name): + + # In this scenario, we will: + # -- download a set of sample contract files + # -- create a Prompt and load a BLING LLM model + # -- parse each contract, extract the relevant passages, and pass questions to a local LLM + + # Main loop - Iterate thru each contract: + # + # 1. parse the document in memory (convert from PDF file into text chunks with metadata) + # 2. filter the parsed text chunks with a "topic" (e.g., "governing law") to extract relevant passages + # 3. package and assemble the text chunks into a model-ready context + # 4. ask three key questions for each contract to the LLM + # 5. print to the screen + # 6. save the results in both json and csv for furthe processing and review. + + # Load the llmware sample files + + print (f"\n > Loading the llmware sample files...") + + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path,"Agreements") + + # Query list - these are the 3 main topics and questions that we would like the LLM to analyze for each contract + + query_list = {"executive employment agreement": "What are the name of the two parties?", + "base salary": "What is the executive's base salary?", + "vacation": "How many vacation days will the executive receive?"} + + # Load the selected model by name that was passed into the function + + print (f"\n > Loading model {model_name}...") + + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + + # Main loop + + for i, contract in enumerate(os.listdir(contracts_path)): + + # excluding Mac file artifact (annoying, but fact of life in demos) + if contract != ".DS_Store": + + print("\nAnalyzing contract: ", str(i+1), contract) + + print("LLM Responses:") + + for key, value in query_list.items(): + + # step 1 + 2 + 3 above - contract is parsed, text-chunked, filtered by topic key, + # ... and then packaged into the prompt + + source = prompter.add_source_document(contracts_path, contract, query=key) + + # step 4 above - calling the LLM with 'source' information already packaged into the prompt + + responses = prompter.prompt_with_source(value, prompt_name="default_with_context") + + # step 5 above - print out to screen + + for r, response in enumerate(responses): + print(key, ":", re.sub("[\n]"," ", response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + + # step 6 above - saving the analysis to jsonl and csv + + # Save jsonl report to jsonl to /prompt_history folder + print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + prompter.save_state() + + # Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + print("csv output saved at: ", csv_output) + + +if __name__ == "__main__": + + # use local cpu model - try the newest - RAG finetune of Phi-3 quantized and packaged in GGUF + model = "bling-phi-3-gguf" + + contract_analysis_on_laptop(model) + +``` +
+ +## 🔥 Solutions + +For project ideas, please see [solutions](https://github.com/llmware-ai/llmware/tree/main/solutions) with dozens of innovative examples. + +### OpenVINO Encoders - ideal for on-device, efficient RAG: + +- 20 OV-optimized encoding models with OVEmbeddingModel class - supports wide of embedding, reranker and classifers, e.g., +- [using_openvino_embedding_model](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/using_openvino_embedding_model.py) +- [using_openvino_reranker_model](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/using_openvino_reranker_model.py) +- [using_openvino_classifier_model](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/using_openvino_classifier_model.py) + +### ONNXRuntime Reranker - use rerankers optimized for Onnxruntime deployment + +- [using_onnx_reranker_model](https://github.com/llmware-ai/llmware/blob/main/solutions/onnxruntime/using_onnx_reranker_models.py) + +### WindowsLocalFoundry integration - use WindowsLocalFoundry models in llmware + +- [using_local_foundry_model](https://github.com/llmware-ai/llmware/blob/main/solutions/onnxruntime/using_local_foundry_models.py) + +### Model Depot - largest out-of-the-box collection of OpenVINO-based LLMs (95+) + +- [Model Depot on Huggingface](https://huggingface.co/collections/llmware/model-depot) +- [using_stream_generation_with_openvino](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/using_openvino_streamer.py) +- [getting_started_with_openvino](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/using_openvino_models.py) + +### Image Generation - Multimedia Bot + +- [multimedia-bot example](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/multimedia_bot.py) + +### ONNXRuntime-QNN - run models on Snapdragon NPU (Windows Arm64) + +- 7 NPU-optimized models 'ready to run' in Model Catalog - see [using-qnn-npu-models example](https://github.com/llmware-ai/llmware/tree/main/solutions/onnxruntime/using-qnn-npu-models.py) + +### Industry-specific Embedding Models for Specialized RAG + +- Check out the LLMware Industry Bert model series - see [industry-bert-models](https://huggingface.co/collections/llmware/industry-bert-models) + +### Audio & Text Processing + +- **Voice Transcription with WhisperCPP** + Start transcription projects with WhisperCPP, featuring tools for sample file usage and famous speeches. + - [Getting started guide](solutions/gguf/using-whisper-cpp-getting-started.py) | [Parsing great speeches](solutions/gguf/parsing_great_speeches.py) | [Demo video](https://youtu.be/5y0ez5ZBpPE?si=KVxsXXtX5TzvlEws) + +- **Natural Language Query to CSV** + Convert natural language queries to CSV with Slim-SQL, supporting custom Postgres tables. + - [Demo video](https://youtu.be/z48z5XOXJJg?si=V-CX1w-7KRioI4Bi) | [End-to-end example](solutions/slim_agents/text2sql-end-to-end-2.py) | [Custom table usage](https://github.com/llmware-ai/llmware/tree/main/solutions/slim_agents/agent_with_custom_tables.py) + +### Multi-Model Agents + +- **Multi-Model Agents with SLIM** + Use SLIM models on CPU for multi-step agents in complex workflows. + - [Demo video](https://www.youtube.com/watch?v=cQfdaTcmBpY) | [Example directory](solutions/slim_agents) + +### Document & OCR Processing + +- **OCR Embedded Document Images** + Extract text systematically from images embedded in documents for enhanced document processing. + - [OCR example](solutions/sources/ocr_embedded_doc_images.py) + +- **Enhanced Document Parsing for PDFs, Word, PowerPoint, and Excel** + Improved text-chunking controls, table extraction, and content parsing. + - [Parsing example](solutions/sources/pdf_parser_new_configs.py) + +- **Optimizing Accuracy of RAG Prompts** + Tutorials for tuning RAG prompt settings for increased accuracy. + - [Settings example](solutions/models/adjusting_sampling_settings.py) | Videos: [Part I](https://youtu.be/7oMTGhSKuNY?si=14mS2pftk7NoKQbC), [Part II](https://youtu.be/iXp1tj-pPjM?si=T4teUAISnSWgtThu) + +New to RAG? [Check out the Fast Start video series](https://www.youtube.com/playlist?list=PL1-dn33KwsmD7SB9iSO6vx4ZLRAWea1DB) + +[Intro to SLIM Function Call Models](https://github.com/llmware-ai/llmware/blob/main/solutions/models/using_function_calls.py) + +## 🌱 Getting Started + +**Step 1 - Install llmware** - `pip3 install llmware` or `pip3 install 'llmware[full]'` + +- [core install](https://github.com/llmware-ai/llmware/blob/main/llmware/requirements.txt) (minimal set of dependencies) +- [full install](https://github.com/llmware-ai/llmware/blob/main/llmware/requirements_extras.txt) (adds to the core with wider set of related python libraries). + +
+Step 2- Go to Examples - Get Started Fast with 100+ 'Cut-and-Paste' Recipes + +## 🔥 Top New Examples 🔥 + +End-to-End Scenario - [**Function Calls with SLIM Extract and Web Services for Financial Research**](https://github.com/llmware-ai/llmware/tree/main/solutions/use_cases/web_services_slim_fx.py) +Analyzing Voice Files - [**Great Speeches with LLM Query and Extract**](https://github.com/llmware-ai/llmware/tree/main/solutions/use_cases/parsing_great_speeches.py) +New to LLMWare - [**Fast Start tutorial series**](https://github.com/llmware-ai/llmware/tree/main/tutorials) +Getting Setup - [**Getting Started**](https://github.com/llmware-ai/llmware/tree/main/tutorials/Getting_Started) +SLIM Examples - [**SLIM Models**](solutions/slim_agents/) + +| Example | Detail | +|-------------|--------------| +| 1. BLING models fast start ([code](solutions/models/bling_fast_start.py) / [video](https://www.youtube.com/watch?v=JjgqOZ2v5oU)) | Get started with fast, accurate, CPU-based models - question-answering, key-value extraction, and basic summarization. | +| 2. Parse and Embed 500 PDF Documents ([code](solutions/embedding/docs2vecs_with_milvus-un_resolutions.py)) | End-to-end example for Parsing, Embedding and Querying UN Resolution documents with Milvus | +| 3. Hybrid Retrieval - Semantic + Text ([code](solutions/sources/dual_pass_with_custom_filter.py)) | Using 'dual pass' retrieval to combine best of semantic and text search | +| 4. Multiple Embeddings with PG Vector ([code](solutions/embedding/using_multiple_embeddings.py) / [video](https://www.youtube.com/watch?v=Bncvggy6m5Q)) | Comparing Multiple Embedding Models using Postgres / PG Vector | +| 5. DRAGON GGUF Models ([code](solutions/models/dragon_gguf_fast_start.py) / [video](https://www.youtube.com/watch?v=BI1RlaIJcsc&t=130s)) | State-of-the-Art 7B RAG GGUF Models. | +| 6. RAG with BLING ([code](solutions/use_cases/contract_analysis_on_laptop_with_bling_models.py) / [video](https://www.youtube.com/watch?v=8aV5p3tErP0)) | Using contract analysis as an example, experiment with RAG for complex document analysis and text extraction using `llmware`'s BLING ~1B parameter GPT model running on your laptop. | +| 7. Master Service Agreement Analysis with DRAGON ([code](solutions/use_cases/msa_processing.py) / [video](https://www.youtube.com/watch?v=Cf-07GBZT68&t=2s)) | Analyzing MSAs using DRAGON YI 6B Model. | +| 8. Streamlit Example ([code](solutions/ui/simple_rag_ui_with_streamlit.py)) | Ask questions to Invoices with UI run inference. | +| 9. Integrating LM Studio ([code](solutions/models/using-open-chat-models.py) / [video](https://www.youtube.com/watch?v=h2FDjUyvsKE&t=101s)) | Integrating LM Studio Models with LLMWare | +| 10. Prompts With Sources ([code](solutions/sources/prompt_with_sources.py)) | Attach wide range of knowledge sources directly into Prompts. | +| 11. Fact Checking ([code](solutions/sources/fact_checking.py)) | Explore the full set of evidence methods in this example script that analyzes a set of contracts. | + + +Check out: [llmware solutions](https://github.com/llmware-ai/llmware/blob/main/solutions/README.md) + +
+ +
+Step 3 - Tutorial Videos - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +🎬 Check out these videos to get started quickly: +- [Document Summarization](https://youtu.be/Ps3W-P9A1m8?si=Rxvst3RJv8ZaOk0L) +- [Bling-3-GGUF Local Chatbot](https://youtu.be/gzzEVK8p3VM?si=8cNn_do0oxSzCEnM) +- [Agent-based Complex Research Analysis](https://youtu.be/y4WvwHqRR60?si=jX3KCrKcYkM95boe) +- [Getting Started with SLIMs (with code)](https://youtu.be/aWZFrTDmMPc?si=lmo98_quo_2Hrq0C) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part I](https://youtu.be/7oMTGhSKuNY?si=_KSjuBnqArvWzYbx) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part II- Code Experiments](https://youtu.be/iXp1tj-pPjM?si=3ZeMgipY0vJDHIMY) +- [SLIM Models Intro](https://www.youtube.com/watch?v=cQfdaTcmBpY) +- [Text2SQL Intro](https://youtu.be/BKZ6kO2XxNo?si=tXGt63pvrp_rOlIP) +- [RAG with BLING on your laptop](https://www.youtube.com/watch?v=JjgqOZ2v5oU) +- [DRAGON-7B-Models](https://www.youtube.com/watch?v=d_u7VaKu6Qk&t=37s) +- [Install and Compare Multiple Embeddings with Postgres and PGVector](https://www.youtube.com/watch?v=Bncvggy6m5Q) +- [Background on GGUF Quantization & DRAGON Model Example](https://www.youtube.com/watch?v=ZJyQIZNJ45E) +- [Using LM Studio Models](https://www.youtube.com/watch?v=h2FDjUyvsKE) +- [Using Ollama Models](https://www.youtube.com/watch?v=qITahpVDuV0) +- [Use any GGUF Model](https://www.youtube.com/watch?v=9wXJgld7Yow) +- [Use small LLMs for RAG for Contract Analysis (feat. LLMWare)](https://www.youtube.com/watch?v=8aV5p3tErP0) +- [Invoice Processing with LLMware](https://www.youtube.com/watch?v=VHZSaBBG-Bo&t=10s) +- [Ingest PDFs at Scale](https://www.youtube.com/watch?v=O0adUfrrxi8&t=10s) +- [Evaluate LLMs for RAG with LLMWare](https://www.youtube.com/watch?v=s0KWqYg5Buk&t=105s) +- [Fast Start to RAG with LLMWare Open Source Library](https://www.youtube.com/watch?v=0naqpH93eEU) +- [Use Retrieval Augmented Generation (RAG) without a Database](https://www.youtube.com/watch?v=tAGz6yR14lw) +- [Pop up LLMWare Inference Server](https://www.youtube.com/watch?v=qiEmLnSRDUA&t=20s) + + +
+ +## ✍️ Working with the llmware Github repository + +The llmware repo can be pulled locally to get access to all the examples, or to work directly with the latest version of the llmware code. + +```bash +git clone git@github.com:llmware-ai/llmware.git +``` + +We have provided a **welcome_to_llmware** automation script in the root of the repository folder. After cloning: +- On Windows command line: `.\welcome_to_llmware_windows.sh` +- On Mac / Linux command line: `sh ./welcome_to_llmware.sh` + +Alternatively, if you prefer to complete setup without the welcome automation script, then the next steps include: + +1. **install requirements.txt** - inside the /llmware path - e.g., ```pip3 install -r llmware/requirements.txt``` + +2. **install requirements_extras.txt** - inside the /llmware path - e.g., ```pip3 install -r llmware/requirements_extras.txt```(Depending upon your use case, you may not need all or any of these installs, but some of these will be used in the examples.) + +3. **run examples** - copy one or more of the example .py files into the root project path. (We have seen several IDEs that will attempt to run interactively from the nested /example path, and then not have access to the /llmware module - the easy fix is to just copy the example you want to run into the root path). + +4. **install vector db** - no-install vector db options include milvus lite, chromadb, faiss and lancedb - which do not require a server install, but do require that you install the python sdk library for that vector db, e.g., `pip3 install pymilvus`, or `pip3 install chromadb`. If you look in [examples/Embedding](https://github.com/llmware-ai/llmware/tree/main/solutions/embedding), you will see examples for getting started with various vector DB, and in the root of the repo, you will see easy-to-get-started docker compose scripts for installing milvus, postgres/pgvector, mongo, qdrant, neo4j, and redis. + + +## Data Store Options + +
+Fast Start: use SQLite3 and ChromaDB (File-based) out-of-the-box - no install required + +```python +from llmware.configs import LLMWareConfig +LLMWareConfig().set_active_db("sqlite") +LLMWareConfig().set_vector_db("chromadb") +``` +
+ +
+Speed + Scale: use MongoDB (text collection) and Milvus (vector db) - install with Docker Compose + +```bash +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose.yaml +docker compose up -d +``` + +```python +from llmware.configs import LLMWareConfig +LLMWareConfig().set_active_db("mongo") +LLMWareConfig().set_vector_db("milvus") +``` + +
+ +
+Postgres: use Postgres for both text collection and vector DB - install with Docker Compose + +```bash +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose-pgvector.yaml +docker compose up -d +``` + +```python +from llmware.configs import LLMWareConfig +LLMWareConfig().set_active_db("postgres") +LLMWareConfig().set_vector_db("postgres") +``` + +
+ +
+Mix-and-Match: LLMWare supports 3 text collection databases (Mongo, Postgres, SQLite) and +10 vector databases (Milvus, PGVector-Postgres, Neo4j, Redis, Mongo-Atlas, Qdrant, Faiss, LanceDB, ChromaDB and Pinecone) + +```bash +# scripts to deploy other options +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose-redis-stack.yaml +``` + +
+ +## Meet our Models + +- **SLIM model series:** small, specialized models fine-tuned for function calling and multi-step, multi-model Agent workflows. +- **DRAGON model series:** Production-grade RAG-optimized 6-9B parameter models - "Delivering RAG on ..." the leading foundation base models. +- **BLING model series:** Small CPU-based RAG-optimized, instruct-following 1B-5B parameter models. +- **Industry BERT models:** out-of-the-box custom trained sentence transformer embedding models fine-tuned for the following industries: Insurance, Contracts, Asset Management, SEC. +- **GGUF Quantization:** we provide 'gguf' and 'tool' versions of many SLIM, DRAGON and BLING models, optimized for CPU deployment. + + +Interested in contributing to llmware? Information on ways to participate can be found in our [Contributors Guide](https://github.com/llmware-ai/llmware/blob/main/repo_docs/CONTRIBUTING.md#contributing-to-llmware). As with all aspects of this project, contributing is governed by our [Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/repo_docs/CODE_OF_CONDUCT.md). + +Questions and discussions are welcome in our [github discussions](https://github.com/llmware-ai/llmware/discussions). + +## 📣 Release notes and Change Log + +For complete history of release notes, please open the Change log tab. + +**Supported Operating Systems**: Windows (x86 and Arm64), MacOS (Metal - M1-M5), Linux (x86, aarch64) +- Linux - support Ubuntu 20+ (glibc 2.31+) +- If you need support for another Linux version, please raise an issue - we will prioritize testing and ensure support. + +**Supported Vector Databases**: Milvus, Postgres (PGVector), Neo4j, Redis, LanceDB, ChromaDB, Qdrant, FAISS, Pinecone, Mongo Atlas Vector Search + +**Supported Text Index Databases**: MongoDB, Postgres, SQLite + + +
+Optional + +- [Docker](https://docs.docker.com/get-docker/) + +- To enable the OCR parsing capabilities, install [Tesseract v5.3.3](https://tesseract-ocr.github.io/tessdoc/Installation.html) and [Poppler v23.10.0](https://poppler.freedesktop.org/) native packages. + +
+ +
+ 🚧 Change Log + +**Thursday, January 1 - v0.4.3 - WIP** + - Updated BaseModel and PromptCatalog classes + - Updated cloud model versions and support for OpenAI, Gemini and Anthropic latest models + - Removed deprecated model classes + - Removed deprecated modules (Dataset Builder and Graph) + - Work-in-Progress for other Model Class and Card updates + - Repo is up-to-date, but not in pip install release - targeted week of January 12, 2026 + +**Monday, March 3 - v0.4.0** + - Updates in GGUF implementation, configs and libs + - Updates in ONNXRuntime implementation and configs + - New Models added to ModelCatalog, including phi-4, Deepseek-Qwen-7B, Deepseek-Qwen-14B, and many others + - Added support for Windows ARM64 + - Changed default active_db to "sqlite" (both mongo and postgres available for production) + - Streamlined dependencies in core requirements.txt and pip install + - 'Extra/optional' dependencies available in requirements_extras.txt and through configurations passed in the pip install process (see setup.py for options) + +**Friday, November 8 - v0.3.9** + - Enhanced Azure OpenAI configuration, including streaming generation + - Removed deprecated parser binaries for Linux aarch64 and Mac x86 + - Added generator option for CustomTable insert rows to provide progress on larger table builds + +**Sunday, October 27 - v0.3.8** + - Integrating Model Depot collection of 100+ OpenVino and ONNX Models into LLMWare default model catalog + - Supporting changes in model classes, model catalog and model configs + +**Sunday, October 6 - v0.3.7** +- Added new model class - OVGenerativeModel - to support the use of models packaged in OpenVino format +- Added new model class - ONNXGenerativeModel - to support use of models packaged in ONNX format +- Getting started with [OpenVino example](https://github.com/llmware-ai/llmware/blob/main/solutions/openvino/using_openvino_models.py) +- Getting started with [ONNX example](https://github.com/llmware-ai/llmware/blob/main/solutions/onnxruntime/using_onnx_models.py) + +**Tuesday, October 1 - v0.3.6** +- Added new prompt and chat templates +- Improved and updated model configurations +- New utility functions for locating and highlighting text matches in search results +- Improved hashing check utility functions + +**Monday, August 26 - v0.3.5** +- Added 10 new BLING+SLIM models to Model Catalog - featuring Qwen2, Phi-3 and Phi-3.5 +- Launched new DRAGON models on Qwen-7B, Yi-9B, Mistral-v0.3, and Llama-3.1 +- Improved GGUF Configs to expand context window +- Added model benchmark performance data to model configs +- Enhanced Utilities hashing functions + +**Monday, July 29 - v03.4** +- Enhanced safety protections for text2sql db reads for LLMfx agents +- New examples - see [example](https://github.com/llmware-ai/llmware/blob/main/solutions/ui/dueling_chatbot.py) +- More Notebook examples - see [notebook examples](https://github.com/llmware-ai/llmware/blob/main/tutorials/notebooks) + +**Monday, July 8 - v03.3** +- Improvements in model configuration options, logging, and various small fixes +- Improved Azure OpenAI configs - see [example](https://github.com/llmware-ai/llmware/blob/main/solutions/models/using-azure-openai.py) + +**Saturday, June 29 - v0.3.2** +- Update to PDF and Office parsers - improvements to configurations in logging and text chunking options + +**Saturday, June 22 - v0.3.1** +- Added module 3 to Fast Start example series [examples 7-9 on Agents & Function Calls](https://github.com/llmware-ai/llmware/tree/main/fast_start) +- Added reranker Jina model for in-memory semantic similarity RAG - see [example](https://github.com/llmware-ai/llmware/tree/main/solutions/embeddings/using_semantic_reranker_with_rag.py) +- Enhanced model fetching parameterization in model loading process +- Added new 'tiny' versions of slim-extract and slim-summary in both Pytorch and GGUF versions - check out 'slim-extract-tiny-tool' and 'slim-summary-tiny-tool' +- [Biz Bot] use case - see [example](https://github.com/llmware-ai/llmware/tree/main/solutions/use_cases/biz_bot.py) and [video](https://youtu.be/4nBYDEjxxTE?si=o6PDPbu0PVcT-tYd) +- Updated numpy reqs <2 and updated yfinance version minimum (>=0.2.38) + +**Tuesday, June 4 - v0.3.0** +- Added support for new Milvus Lite embedded 'no-install' database - see [example](https://github.com/llmware-ai/llmware/tree/main/solutions/embeddings/using_milvus_lite.py). +- Added two new SLIM models to catalog and agent processes - ['q-gen'](https://github.com/llmware-ai/llmware/tree/main/solutions/slim_agents/using-slim-q-gen.py) and ['qa-gen'](https://github.com/llmware-ai/llmware/tree/main/solutions/slim_agents/using-slim-qa-gen.py) +- Updated model class instantiation to provide more extensibility to add new classes in different modules +- New welcome_to_llmware.sh and welcome_to_llmware_windows.sh fast install scripts +- Enhanced Model class base with new configurable post_init and register methods +- Created InferenceHistory to track global state of all inferences completed +- Multiple improvements and updates to logging at module level +- Note: starting with v0.3.0, pip install provides two options - a base minimal install `pip3 install llmware` which will support most use cases, and a larger install `pip3 install 'llmware[full]'` with other commonly-used libraries. + +**Wednesday, May 22 - v0.2.15** +- Improvements in Model class handling of Pytorch and Transformers dependencies (just-in-time loading, if needed) + +**Saturday, May 18 - v0.2.14** +- New OCR image parsing methods with [example](https://github.com/llmware-ai/llmware/tree/main/solutions/use_cases/slicing_and_dicing_office_docs.py) +- Adding first part of logging improvements (WIP) in Configs and Models. +- New embedding model added to catalog - industry-bert-loans. +- Updates to model import methods and configurations. + +**Sunday, May 12 - v0.2.13** +- New GGUF streaming method with [basic example](https://github.com/llmware-ai/llmware/tree/main/solutions/gguf/gguf_streaming.py) and [phi3 local chatbot](https://github.com/llmware-ai/llmware/tree/main/solutions/ui/gguf_streaming_chatbot.py) +- Significant cleanups in ancillary imports and dependencies to reduce install complexity - note: the updated requirements.txt and setup.py files. +- Defensive code to provide informative warning of any missing dependencies in specialized parts of the code, e.g., OCR, Web Parser. +- Updates of tests, notice and documentation. +- OpenAIConfigs created to support Azure OpenAI. + +**Sunday, May 5 - v0.2.12 Update** +- Launched ["bling-phi-3"](https://huggingface.co/llmware/bling-phi-3) and ["bling-phi-3-gguf"](https://huggingface.co/llmware/bling-phi-3-gguf) in ModelCatalog - newest and most accurate BLING/DRAGON model +- Added support for Python 3.12 +- Deprecated faiss and replaced with 'no-install' chromadb in Fast Start examples +- Refactored Datasets, Graph and Web Services classes +- Updated Voice parsing with WhisperCPP into Library + +**Monday, April 29 - v0.2.11 Update** +- Updates to gguf libs for Phi-3 and Llama-3 +- Integrated WhisperCPP Model class and prebuilt shared libraries - [getting-started-example](https://github.com/llmware-ai/llmware/tree/main/solutions/gguf/using-whisper-cpp-getting-started.py) +- New voice sample files for testing - [example](https://github.com/llmware-ai/llmware/tree/main/solutions/gguf/using-whisper-cpp-sample-files.py) +- Improved CUDA detection on Windows and safety checks for older Mac OS versions + +**Monday, April 22 - v0.2.10 Update** +- Updates to Agent class to support Natural Language queries of Custom Tables on Postgres [example](https://github.com/llmware-ai/llmware/tree/main/solutions/use_cases/agent_with_custom_tables.py) +- New Agent API endpoint implemented with LLMWare Inference Server and new Agent capabilities [example](https://github.com/llmware-ai/llmware/tree/main/solutions/slim_agents/agent_api_endpoint.py) + +**Tuesday, April 16 - v0.2.9 Update** +- New CustomTable class to rapidly create custom DB tables in conjunction with LLM-based workflows. +- Enhanced methods for converting CSV and JSON/JSONL files into DB tables. +- See new examples [Creating Custom Table example](https://github.com/llmware-ai/llmware/tree/main/solutions/sources/create_custom_table-1.py) + +**Tuesday, April 9 - v0.2.8 Update** +- Office Parser (Word Docx, Powerpoint PPTX, and Excel XLSX) - multiple improvements - new libs + Python method. +- Includes: several fixes, improved text chunking controls, header text extraction and configuration options. +- Generally, new office parser options conform with the new PDF parser options. +- Please see [Office Parsing Configs example](https://github.com/llmware-ai/llmware/tree/main/solutions/sources/office_parser_new_configs.py) + +**Wednesday, April 3 - v0.2.7 Update** +- PDF Parser - multiple improvements - new libs + Python methods. +- Includes: UTF-8 encoding for European languages. +- Includes: Better text chunking controls, header text extraction and configuration options. +- Please see [PDF Parsing Configs example](https://github.com/llmware-ai/llmware/tree/main/solutions/sources/pdf_parser_new_configs.py) for more details. +- Note: deprecating support for aarch64-linux (will use 0.2.6 parsers). Full support going forward for Linux Ubuntu20+ on x86_64 + with CUDA. + +**Friday, March 22 - v0.2.6 Update** +- New SLIM models: summary, extract, xsum, boolean, tags-3b, and combo sentiment-ner. +- New logit and sampling analytics. +- New SLIM examples showing how to use the new models. + +**Thursday, March 14 - v0.2.5 Update** +- Improved support for GGUF on CUDA (Windows and Linux), with new prebuilt binaries and exception handling. +- Enhanced model configuration options (sampling, temperature, top logit capture). +- Added full back-level support for Ubuntu 20+ with parsers and GGUF engine. +- Support for new Anthropic Claude 3 models. +- New retrieval methods: document_lookup and aggregate_text. +- New model: bling-stablelm-3b-tool - fast, accurate 3b quantized question-answering model - one of our new favorites. + +**Wednesday, February 28 - v0.2.4 Update** +- Major upgrade of GGUF Generative Model class - support for Stable-LM-3B, CUDA build options, and better control over sampling strategies. +- Note: new GGUF llama.cpp built libs packaged with build starting in v0.2.4. +- Improved GPU support for HF Embedding Models. + +**Friday, February 16 - v0.2.3 Update** +- Added 10+ embedding models to ModelCatalog - nomic, jina, bge, gte, ember and uae-large. +- Updated OpenAI support >=1.0 and new text-3 embedding models. +- SLIM model keys and output_values now accessible in ModelCatalog. +- Updating encodings to 'utf-8-sig' to better handle txt/csv files with bom. + +**Latest Updates - 19 Jan 2024 - llmware v0.2.0** + - Added new database integration options - Postgres and SQlite + - Improved status update and parser event logging options for parallelized parsing + - Significant enhancements to interactions between Embedding + Text collection databases + - Improved error exception handling in loading dynamic modules + +**Latest Updates - 15 Jan 2024: llmware v0.1.15** + - Enhancements to dual pass retrieval queries + - Expanded configuration objects and options for endpoint resources + +**Latest Updates - 30 Dec 2023: llmware v0.1.14** + - Added support for Open Chat inference servers (compatible with OpenAI API) + - Improved capabilities for multiple embedding models and vector DB configurations + - Added docker-compose install scripts for PGVector and Redis vector databases + - Added 'bling-tiny-llama' to model catalog + +**Latest Updates - 22 Dec 2023: llmware v0.1.13** + - Added 3 new vector databases - Postgres (PG Vector), Redis, and Qdrant + - Improved support for integrating sentence transformers directly in the model catalog + - Improvements in the model catalog attributes + - Multiple new Examples in Models & Embeddings, including GGUF, Vector database, and model catalog + +- **17 Dec 2023: llmware v0.1.12** + - dragon-deci-7b added to catalog - RAG-finetuned model on high-performance new 7B model base from Deci + - New GGUFGenerativeModel class for easy integration of GGUF Models + - Adding prebuilt llama_cpp / ctransformer shared libraries for Mac M1, Mac x86, Linux x86 and Windows + - 3 DRAGON models packaged as Q4_K_M GGUF models for CPU laptop use (dragon-mistral-7b, dragon-llama-7b, dragon-yi-6b) + - 4 leading open source chat models added to default catalog with Q4_K_M + +- **8 Dec 2023: llmware v0.1.11** + - New fast start examples for high volume Document Ingestion and Embeddings with Milvus. + - New LLMWare 'Pop up' Inference Server model class and example script. + - New Invoice Processing example for RAG. + - Improved Windows stack management to support parsing larger documents. + - Enhancing debugging log output mode options for PDF and Office parsers. + +- **30 Nov 2023: llmware v0.1.10** + - Windows added as a supported operating system. + - Further enhancements to native code for stack management. + - Minor defect fixes. + +- **24 Nov 2023: llmware v0.1.9** + - Markdown (.md) files are now parsed and treated as text files. + - PDF and Office parser stack optimizations which should avoid the need to set ulimit -s. + - New llmware_models_fast_start.py example that allows discovery and selection of all llmware HuggingFace models. + - Native dependencies (shared libraries and dependencies) now included in repo to faciliate local development. + - Updates to the Status class to support PDF and Office document parsing status updates. + - Minor defect fixes including image block handling in library exports. + +- **17 Nov 2023: llmware v0.1.8** + - Enhanced generation performance by allowing each model to specific the trailing space parameter. + - Improved handling for eos_token_id for llama2 and mistral. + - Improved support for Hugging Face dynamic loading + - New examples with the new llmware DRAGON models. + +- **14 Nov 2023: llmware v0.1.7** + - Moved to Python Wheel package format for PyPi distribution to provide seamless installation of native dependencies on all supported platforms. + - ModelCatalog enhancements: + - OpenAI update to include newly announced ‘turbo’ 4 and 3.5 models. + - Cohere embedding v3 update to include new Cohere embedding models. + - BLING models as out-of-the-box registered options in the catalog. They can be instantiated like any other model, even without the “hf=True” flag. + - Ability to register new model names, within existing model classes, with the register method in ModelCatalog. + - Prompt enhancements: + - “evidence_metadata” added to prompt_main output dictionaries allowing prompt_main responses to be plug into the evidence and fact-checking steps without modification. + - API key can now be passed directly in a prompt.load_model(model_name, api_key = “[my-api-key]”) + - LLMWareInference Server - Initial delivery: + - New Class for LLMWareModel which is a wrapper on a custom HF-style API-based model. + - LLMWareInferenceServer is a new class that can be instantiated on a remote (GPU) server to create a testing API-server that can be integrated into any Prompt workflow. + +- **03 Nov 2023: llmware v0.1.6** + - Updated packaging to require mongo-c-driver 1.24.4 to temporarily workaround segmentation fault with mongo-c-driver 1.25. + - Updates in python code needed in anticipation of future Windows support. + +- **27 Oct 2023: llmware v0.1.5** + - Four new example scripts focused on RAG workflows with small, fine-tuned instruct models that run on a laptop (`llmware` [BLING](https://huggingface.co/llmware) models). + - Expanded options for setting temperature inside a prompt class. + - Improvement in post processing of Hugging Face model generation. + - Streamlined loading of Hugging Face generative models into prompts. + - Initial delivery of a central status class: read/write of embedding status with a consistent interface for callers. + - Enhanced in-memory dictionary search support for multi-key queries. + - Removed trailing space in human-bot wrapping to improve generation quality in some fine-tuned models. + - Minor defect fixes, updated test scripts, and version update for Werkzeug to address [dependency security alert](https://github.com/llmware-ai/llmware/security/dependabot/2). +- **20 Oct 2023: llmware v0.1.4** + - GPU support for Hugging Face models. + - Defect fixes and additional test scripts. +- **13 Oct 2023: llmware v0.1.3** + - MongoDB Atlas Vector Search support. + - Support for authentication using a MongoDB connection string. + - Document summarization methods. + - Improvements in capturing the model context window automatically and passing changes in the expected output length. + - Dataset card and description with lookup by name. + - Processing time added to model inference usage dictionary. + - Additional test scripts, examples, and defect fixes. +- **06 Oct 2023: llmware v0.1.1** + - Added test scripts to the github repository for regression testing. + - Minor defect fixes and version update of Pillow to address [dependency security alert](https://github.com/llmware-ai/llmware/security/dependabot/1). +- **02 Oct 2023: llmware v0.1.0** 🔥 Initial release of llmware to open source!! 🔥 + + +
+

+ ⬆️ Back to Top +

+ +## 🤓 Read our White Papers + + +- **Revolutionizing AI Deployment: Unleashing AI Acceleration with Intel's AI PCs and Model HQ by LLMWare** [AI PC Model HQ.pdf](https://github.com/user-attachments/files/18024139/AI.PC.Model.HQ.pdf) +- **Revultionizing AI Deployment (Intel Abstract Version)** [LNL White paper (Abstract Version) final.pdf](https://github.com/user-attachments/files/18281644/LNL.White.paper.Abstract.Version.final.pdf) + +- **Accelerating AI Powered Productivity with AI PCs** [Laptop.Performance.WP.Final (10).pdf](https://github.com/user-attachments/files/18024294/Laptop.Performance.WP.Final.10.pdf) + +## Intel Joint Solutions + +- **Arrow Lake** +[IPA.Optimization.Summary.LLMWare (1).pdf](https://github.com/user-attachments/files/18292873/IPA.Optimization.Summary.LLMWare.1.pdf) + +## About Model HQ + - **Privacy Policy** [AI.BLOKS.PRIVACY.POLICY.1.3.25.pdf](https://github.com/user-attachments/files/19289355/AI.BLOKS.PRIVACY.POLICY.1.3.25.pdf) + +- **Terms of Service** [AI.Bloks.Terms.of.Service.3.3.25.pdf](https://github.com/user-attachments/files/19289545/AI.Bloks.Terms.of.Service.3.3.25.pdf) + +- **Acceptable Use Policy**[Acceptable Use Policy for Model HQ by AI BLOKS LLC.docx](https://github.com/user-attachments/files/18291481/Acceptable.Use.Policy.for.Model.HQ.by.AI.BLOKS.LLC.docx) + diff --git a/README.wehub.md b/README.wehub.md new file mode 100644 index 0000000..d7b15c9 --- /dev/null +++ b/README.wehub.md @@ -0,0 +1,7 @@ +# WeHub 来源说明 + +- 原始项目：`llmware-ai/llmware` +- 原始仓库：https://github.com/llmware-ai/llmware +- 导入方式：上游默认分支的最新快照 +- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准 +- 本文件仅用于记录来源，不代表 WeHub 是原项目作者 diff --git a/docs/1.0.4 b/docs/1.0.4 new file mode 100644 index 0000000..dd2aba7 --- /dev/null +++ b/docs/1.0.4 @@ -0,0 +1,94 @@ +Fetching gem metadata from https://rubygems.org/.......... +Resolving dependencies... +Resolving dependencies... +Fetching rake 13.2.1 +Installing rake 13.2.1 +Fetching public_suffix 6.0.1 +Fetching bigdecimal 3.1.8 +Fetching colorator 1.1.0 +Fetching concurrent-ruby 1.3.4 +Fetching eventmachine 1.2.7 +Fetching uri 1.1.1 (was 1.0.1) +Fetching http_parser.rb 0.8.0 +Fetching json 2.8.1 +Installing colorator 1.1.0 +Fetching logger 1.6.1 +Installing uri 1.1.1 (was 1.0.1) +Fetching ffi 1.17.0 +Installing bigdecimal 3.1.8 with native extensions +Installing public_suffix 6.0.1 +Fetching forwardable-extended 2.6.0 +Installing logger 1.6.1 +Fetching rb-fsevent 0.11.2 +Installing json 2.8.1 with native extensions +Installing eventmachine 1.2.7 with native extensions +Installing forwardable-extended 2.6.0 +Fetching rexml 3.3.9 +Installing rb-fsevent 0.11.2 +Fetching liquid 4.0.4 +Installing liquid 4.0.4 +Installing rexml 3.3.9 +Installing http_parser.rb 0.8.0 with native extensions +Fetching mercenary 0.4.0 +Fetching rouge 4.4.0 +Installing concurrent-ruby 1.3.4 +Installing mercenary 0.4.0 +Fetching safe_yaml 1.0.5 +Installing safe_yaml 1.0.5 +Fetching unicode-display_width 2.6.0 +Fetching webrick 1.9.0 +Installing unicode-display_width 2.6.0 +Fetching net-http 0.5.0 +Installing ffi 1.17.0 with native extensions +Installing webrick 1.9.0 +Installing net-http 0.5.0 +Fetching addressable 2.8.7 +Installing rouge 4.4.0 +Fetching pathutil 0.16.2 +Installing pathutil 0.16.2 +Installing addressable 2.8.7 +Fetching kramdown 2.4.0 +Fetching i18n 1.14.6 +Installing kramdown 2.4.0 +Installing i18n 1.14.6 +Fetching terminal-table 3.0.2 +Installing terminal-table 3.0.2 +Fetching faraday-net_http 3.3.0 +Installing faraday-net_http 3.3.0 +Fetching kramdown-parser-gfm 1.1.0 +Installing kramdown-parser-gfm 1.1.0 +Fetching faraday 2.12.0 +Installing faraday 2.12.0 +Fetching sawyer 0.9.2 +Installing sawyer 0.9.2 +Fetching octokit 6.1.1 +Installing octokit 6.1.1 +Fetching google-protobuf 4.28.3 +Installing google-protobuf 4.28.3 with native extensions +Fetching sass-embedded 1.80.6 (x64-mingw-ucrt) +Fetching em-websocket 0.5.3 +Installing em-websocket 0.5.3 +Installing sass-embedded 1.80.6 (x64-mingw-ucrt) +Fetching jekyll-sass-converter 3.0.0 +Installing jekyll-sass-converter 3.0.0 +Fetching rb-inotify 0.11.1 +Installing rb-inotify 0.11.1 +Fetching listen 3.9.0 +Installing listen 3.9.0 +Fetching jekyll-watch 2.2.1 +Installing jekyll-watch 2.2.1 +Fetching jekyll 4.3.4 +Installing jekyll 4.3.4 +Fetching jekyll-default-layout 0.1.5 +Fetching jekyll-github-metadata 2.16.1 +Fetching jekyll-seo-tag 2.8.0 +Fetching jekyll-include-cache 0.2.1 +Installing jekyll-include-cache 0.2.1 +Installing jekyll-github-metadata 2.16.1 +Installing jekyll-seo-tag 2.8.0 +Installing jekyll-default-layout 0.1.5 +Fetching just-the-docs 0.7.0 +Installing just-the-docs 0.7.0 +Bundle updated! +Post-install message from i18n: +PSA: I18n will be dropping support for Ruby < 3.2 in the next major release (April 2025), due to Ruby's end of life for 3.1 and below (https://endoflife.date/ruby). Please upgrade to Ruby 3.2 or newer by April 2025 to continue using future versions of this gem. diff --git a/docs/3.4.2 b/docs/3.4.2 new file mode 100644 index 0000000..6c6bc7a --- /dev/null +++ b/docs/3.4.2 @@ -0,0 +1,5 @@ +Fetching gem metadata from https://rubygems.org/.......... +Resolving dependencies... +Resolving dependencies... +Using rexml 3.4.4 (was 3.3.9) +Bundle updated! diff --git a/docs/Gemfile b/docs/Gemfile new file mode 100644 index 0000000..5287161 --- /dev/null +++ b/docs/Gemfile @@ -0,0 +1,26 @@ +source 'https://rubygems.org' + + +# +# Gems that are locked to a version. +# + +ruby ">=2.7" + +gem "jekyll", "~> 4.3.2" # installed by `gem jekyll` +gem "just-the-docs", "0.7.0" # pinned to the current release +gem "rexml", ">= 3.3.9" # pinned to that version to fix security alert +gem "google-protobuf", ">= 3.25.5" + +# +# Gems that are only loaded if they are configured correctly. +# +gem 'jekyll-default-layout' + + +# +# Gems that loaded irrespective of site configuration. +# +gem 'jekyll-seo-tag', group: :jekyll_plugins +gem 'jekyll-include-cache', group: :jekyll_plugins +gem 'jekyll-github-metadata', ">= 2.15", group: :jekyll_plugins diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock new file mode 100644 index 0000000..9d1a93c --- /dev/null +++ b/docs/Gemfile.lock @@ -0,0 +1,212 @@ +GEM + remote: https://rubygems.org/ + specs: + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) + bigdecimal (3.1.8) + colorator (1.1.0) + concurrent-ruby (1.3.4) + em-websocket (0.5.3) + eventmachine (>= 0.12.9) + http_parser.rb (~> 0) + eventmachine (1.2.7) + faraday (2.12.0) + faraday-net_http (>= 2.0, < 3.4) + json + logger + faraday-net_http (3.3.0) + net-http + ffi (1.17.0) + ffi (1.17.0-aarch64-linux-gnu) + ffi (1.17.0-aarch64-linux-musl) + ffi (1.17.0-arm-linux-gnu) + ffi (1.17.0-arm-linux-musl) + ffi (1.17.0-arm64-darwin) + ffi (1.17.0-x86-linux-gnu) + ffi (1.17.0-x86-linux-musl) + ffi (1.17.0-x86_64-darwin) + ffi (1.17.0-x86_64-linux-gnu) + ffi (1.17.0-x86_64-linux-musl) + forwardable-extended (2.6.0) + google-protobuf (4.28.3) + bigdecimal + rake (>= 13) + google-protobuf (4.28.3-aarch64-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.28.3-arm64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.28.3-x86-linux) + bigdecimal + rake (>= 13) + google-protobuf (4.28.3-x86_64-darwin) + bigdecimal + rake (>= 13) + google-protobuf (4.28.3-x86_64-linux) + bigdecimal + rake (>= 13) + http_parser.rb (0.8.0) + i18n (1.14.6) + concurrent-ruby (~> 1.0) + jekyll (4.3.4) + addressable (~> 2.4) + colorator (~> 1.0) + em-websocket (~> 0.5) + i18n (~> 1.0) + jekyll-sass-converter (>= 2.0, < 4.0) + jekyll-watch (~> 2.0) + kramdown (~> 2.3, >= 2.3.1) + kramdown-parser-gfm (~> 1.0) + liquid (~> 4.0) + mercenary (>= 0.3.6, < 0.5) + pathutil (~> 0.9) + rouge (>= 3.0, < 5.0) + safe_yaml (~> 1.0) + terminal-table (>= 1.8, < 4.0) + webrick (~> 1.7) + jekyll-default-layout (0.1.5) + jekyll (>= 3.0, < 5.0) + jekyll-github-metadata (2.16.1) + jekyll (>= 3.4, < 5.0) + octokit (>= 4, < 7, != 4.4.0) + jekyll-include-cache (0.2.1) + jekyll (>= 3.7, < 5.0) + jekyll-sass-converter (3.0.0) + sass-embedded (~> 1.54) + jekyll-seo-tag (2.8.0) + jekyll (>= 3.8, < 5.0) + jekyll-watch (2.2.1) + listen (~> 3.0) + json (2.8.1) + just-the-docs (0.7.0) + jekyll (>= 3.8.5) + jekyll-include-cache + jekyll-seo-tag (>= 2.0) + rake (>= 12.3.1) + kramdown (2.4.0) + rexml + kramdown-parser-gfm (1.1.0) + kramdown (~> 2.0) + liquid (4.0.4) + listen (3.9.0) + rb-fsevent (~> 0.10, >= 0.10.3) + rb-inotify (~> 0.9, >= 0.9.10) + logger (1.6.1) + mercenary (0.4.0) + net-http (0.5.0) + uri + octokit (6.1.1) + faraday (>= 1, < 3) + sawyer (~> 0.9) + pathutil (0.16.2) + forwardable-extended (~> 2.6) + public_suffix (6.0.1) + rake (13.2.1) + rb-fsevent (0.11.2) + rb-inotify (0.11.1) + ffi (~> 1.0) + rexml (3.4.4) + rouge (4.4.0) + safe_yaml (1.0.5) + sass-embedded (1.80.6) + google-protobuf (~> 4.28) + rake (>= 13) + sass-embedded (1.80.6-aarch64-linux-android) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-aarch64-linux-gnu) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-aarch64-linux-musl) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-aarch64-mingw-ucrt) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-arm-linux-androideabi) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-arm-linux-gnueabihf) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-arm-linux-musleabihf) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-arm64-darwin) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-riscv64-linux-android) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-riscv64-linux-gnu) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-riscv64-linux-musl) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x64-mingw-ucrt) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86-cygwin) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86-linux-android) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86-linux-gnu) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86-linux-musl) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86-mingw-ucrt) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86_64-cygwin) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86_64-darwin) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86_64-linux-android) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86_64-linux-gnu) + google-protobuf (~> 4.28) + sass-embedded (1.80.6-x86_64-linux-musl) + google-protobuf (~> 4.28) + sawyer (0.9.2) + addressable (>= 2.3.5) + faraday (>= 0.17.3, < 3) + terminal-table (3.0.2) + unicode-display_width (>= 1.1.1, < 3) + unicode-display_width (2.6.0) + uri (1.1.1) + webrick (1.9.0) + +PLATFORMS + aarch64-linux + aarch64-linux-android + aarch64-linux-gnu + aarch64-linux-musl + aarch64-mingw-ucrt + arm-linux-androideabi + arm-linux-gnu + arm-linux-gnueabihf + arm-linux-musl + arm-linux-musleabihf + arm64-darwin + riscv64-linux-android + riscv64-linux-gnu + riscv64-linux-musl + ruby + x64-mingw-ucrt + x86-cygwin + x86-linux + x86-linux-android + x86-linux-gnu + x86-linux-musl + x86-mingw-ucrt + x86_64-cygwin + x86_64-darwin + x86_64-linux + x86_64-linux-android + x86_64-linux-gnu + x86_64-linux-musl + +DEPENDENCIES + google-protobuf (>= 3.25.5) + jekyll (~> 4.3.2) + jekyll-default-layout + jekyll-github-metadata (>= 2.15) + jekyll-include-cache + jekyll-seo-tag + just-the-docs (= 0.7.0) + rexml (>= 3.3.9) + +RUBY VERSION + ruby 3.3.6p108 + +BUNDLED WITH + 4.0.3 diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..4130ff9 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,54 @@ +# Site settings +# These are used to personalize your new site. If you look in the HTML files, +# you will see them accessed via {{ site.title }}, {{ site.github_repo }}, and so on. +# You can create any custom variable you would like, and they will be accessible +# in the templates via {{ site.myvariable }}. +title: llmware +description: llmware is an enterprise-grade LLM-based development framework, including tools and fine-tuned models. +theme: just-the-docs +url: https://llmware-ai.github.io/llmware/ +baseurl: "" # the subpath of your site +favicon_ico: "/assets/images/favicon.ico" +repository: llmware-ai/llmware # for github-metadata + +# Enable or disable heading anchors +heading_anchors: true + +# Aux links for the upper right navigation +aux_links: + llmware repository: https://github.com/llmware-ai/llmware + +lsi: false +safe: true +highlighter: rouge + +gist: + noscript: false +kramdown: + math_engine: mathjax + syntax_highlighter: rouge + + +callouts_level: quiet # or loud +callouts: + highlight: + color: yellow + important: + title: Important + color: blue + new: + title: New + color: green + note: + title: Note + color: purple + warning: + title: Warning + color: red + + +plugins: + - jekyll-default-layout + - jekyll-seo-tag + - jekyll-github-metadata + - jekyll-include-cache diff --git a/docs/_includes/footer_custom.html b/docs/_includes/footer_custom.html new file mode 100644 index 0000000..a1202b7 --- /dev/null +++ b/docs/_includes/footer_custom.html @@ -0,0 +1,43 @@ + + +

+ © 2023-{{ "now" | date: "%Y" }} AI Bloks +

diff --git a/docs/_includes/head_custom.html b/docs/_includes/head_custom.html new file mode 100644 index 0000000..ac555a8 --- /dev/null +++ b/docs/_includes/head_custom.html @@ -0,0 +1,65 @@ + + + + diff --git a/docs/assets/images/favicon.ico b/docs/assets/images/favicon.ico new file mode 100644 index 0000000..b596112 Binary files /dev/null and b/docs/assets/images/favicon.ico differ diff --git a/docs/assets/images/hf-logo.svg b/docs/assets/images/hf-logo.svg new file mode 100644 index 0000000..ab959d1 --- /dev/null +++ b/docs/assets/images/hf-logo.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/docs/assets/images/llmware_logo_color_cropped.png b/docs/assets/images/llmware_logo_color_cropped.png new file mode 100644 index 0000000..a618d5b Binary files /dev/null and b/docs/assets/images/llmware_logo_color_cropped.png differ diff --git a/docs/community/community.md b/docs/community/community.md new file mode 100644 index 0000000..5972e7b --- /dev/null +++ b/docs/community/community.md @@ -0,0 +1,105 @@ +--- +layout: default +title: Community +nav_order: 6 +has_children: true +description: community resources, getting help and sharing ideas +permalink: /community +--- + +# Community + +Welcome to the llmware community! We are on a mission to pioneer the use of small language models as transformational tools +in the enterprise to automate workflows and knowledge-based processes cost-effectively, securely and with high quality. We believe that the +secret is increasing out that small models can be extremely effective, but require a lot of attention to detail in building scalable data pipelines +and fine-tuning both models and end-to-end workflows. + +We are open to both the most advanced machine learning researchers and the beginning developer just learning python. + +We publish a wide range of examples, use cases and tutorial videos, and are always looking for feedback, new ideas and contributors. + + + +{: .note} +> The contributions to `llmware` are governed by our [Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md). + +{: .warning} +> Have you found a security issue? Then please jump to [Security Vulnerabilities](#security-vulnerabilities). + +On this page, we provide information ``llmware`` contributions. +There are **two ways** on how you can contribute. +The first is by making **code contributions**, and the second by making contributions to the **documentation**. +Please look at our [contribution suggestions](#how-can-you-contribute) if you need inspiration, or take a look at [open issues](#open-issues). + +Contributions to `llmware` are welcome from everyone. +Our goal is to make the process simple, transparent, and straightforward. +We are happy to receive suggestions on how the process can be improved. + +## How can you contribute? + +{: .note} +> If you have never contributed before look for issues with the tag [``good first issue``](https://github.com/llmware-ai/llmware/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). + +The most usual ways to contribute is to add new features, fix bugs, add tests, or add documentation. +You can visit the [issues](https://github.com/llmware-ai/llmware/issues) site of the project and search for tags such as +``bug``, ``enhancement``, ``documentation``, or ``test``. + + +Here is a non exhaustive list of contributions you can make. + +1. Code refactoring +2. Add new text data bases +3. Add new vector data bases +4. Fix bugs +5. Add usage examples (see for example the issues [jupyter notebook - more examples and better support](https://github.com/llmware-ai/llmware/issues/508) and [google colab examples and start up scripts](https://github.com/llmware-ai/llmware/issues/507)) +6. Add experimental features +7. Improve code quality +8. Improve documentation in the docs (what you are reading right now) +9. Improve documentation by adding or updating docstrings in modules, classes, methods, or functions (see for example [Add docstrings](https://github.com/llmware-ai/llmware/issues/219)) +10. Improve test coverage +11. Answer questions in our [Discord channel](https://discord.gg/MhZn5Nc39h), especially in the [technical support forum](https://discord.com/channels/1179245642770559067/1218498778915672194) +12. Post projects in which you use ``llmware`` in our Discord forum [made with llmware](https://discord.com/channels/1179245642770559067/1218567269471486012), ideally with a link to a public GitHub repository + +## Open Issues +If you're interested in existing issues, you can + +- Look for issues, if you are a new to the project, look for issues with the `good first issue` label. +- Provide answers for questions in our [GitHub discussions](https://github.com/llmware-ai/llmware/discussions) +- Provide help for bug or enhancement issues. + - Ask questions, reproduce the issues, or provide solutions. + - Pull a request to fix the issue. + + + +## Security Vulnerabilities +**If you believe you've found a security vulnerability, then please _do not_ submit an issue ticket or pull request or otherwise publicly disclose the issue.** +Please follow the process at [Reporting a Vulnerability](https://github.com/llmware-ai/llmware/blob/main/Security.md) + + + +## GitHub workflow + +We follow the [``fork-and-pull``](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) Git workflow. + +1. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the repository on GitHub. +2. Clone your fork to your local machine with `git clone git@github.com:/llmware.git`. +3. Create a branch with `git checkout -b my-topic-branch`. +4. Run the test suite by navigating to the tests/ folder and running ```./run-tests.py -s``` to ensure there are no failures +5. [Commit](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/committing-changes-to-a-pull-request-branch-created-from-a-fork) changes to your own branch, then push to GitHub with `git push origin my-topic-branch`. +6. Submit a [pull request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) so that we can review your changes. + +Remember to [synchronize your forked repository](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo#keep-your-fork-synced) _before_ submitting proposed changes upstream. If you have an existing local repository, please update it before you start, to minimize the chance of merge conflicts. + +```shell +git remote add upstream git@github.com:llmware-ai/llmware.git +git fetch upstream +git checkout upstream/main -b my-topic-branch +``` + +## Community +Questions and discussions are welcome in any shape or form. +Please fell free to join our community on our discord channel, on which we are active daily. +You are also welcome if you just want to post an idea! + +- [Discord Channel](https://discord.gg/MhZn5Nc39h) +- [GitHub discussions](https://github.com/llmware-ai/llmware/discussions) diff --git a/docs/community/faq.md b/docs/community/faq.md new file mode 100644 index 0000000..9a78c7c --- /dev/null +++ b/docs/community/faq.md @@ -0,0 +1,191 @@ +--- +layout: default +title: FAQ +parent: Community +nav_order: 1 +description: overview of the major modules and classes of LLMWare +permalink: /community/faq +--- +# Frequently Asked Questions (FAQ) + + +### How can I set the chunk size? +#### "I want to parse my documents into smaller chunks" +You can set the chunk size with the ``chunk_size`` parameter of the ``add_files`` method. + +The ``add_files`` method from the ``Library`` class has a ``chunk_size`` parameter that controls the chunk size. +The method in addition has a parameter to control the maximum chunk size with ``max_chunk_size``. +These two parameters are passed on to the ``Parser`` class. +In the following example, we add the same files with different chunk sizes to the library ``chunk_size_example``. +```python +from pathlib import Path + +from llmware.library import Library + + +path_to_my_library_files = Path('~/llmware_data/sample_files/Agreements') + +my_library = Library().create_new_library(library_name='chunk_size_example') +my_library.add_files(input_folder_path=path_to_my_library_files, chunk_size=400) +my_library.add_files(input_folder_path=path_to_my_library_files, chunk_size=600) +``` + +### How can I set the embedding store? +#### "I want to use a specific embedding store" +You can set the embedding store with the ``vector_db`` parameter of the ``install_new_embedding`` method, which you call on a ``Library`` object each time you want to create an embedding for a *library*. + +The ``install_new_embedding`` method from the ``Library`` class has a ``vector_db`` parameter that sets the embedding store. +At the moment of this writing, *LLMWare* supports the embedding stores [chromadb](https://github.com/chroma-core/chroma), [neo4j](https://github.com/neo4j/neo4j), [milvus](https://github.com/milvus-io/milvus), [pg_vector](https://github.com/pgvector/pgvector), [postgres](https://github.com/postgres/postgres), [redis](https://github.com/redis/redis), [pinecone](https://www.pinecone.io/), [faiss](https://github.com/facebookresearch/faiss), [qdrant](https://github.com/qdrant/qdrant), [mongo atlas](https://www.mongodb.com/products/platform/atlas-database), and [lancedb](https://github.com/lancedb/lancedb). +In the following example, we create the same embeddings three times for the same library, but store them in three different embedding stores. +```python +import logging +from pathlib import Path + +from llmware.configs import LLMWareConfig +from llmware.library import Library + + +logging.info(f'Currently supported embedding stores: {LLMWareConfig().get_supported_vector_db()}') + +library = Library().create_new_library(library_name='embedding_store_example') +library.add_files(input_foler_path=Path('~/llmware_data/sample_files/Agreements')) + +library.install_new_embedding(vector_db="pg_vector") +library.install_new_embedding(vector_db="milvus") +library.install_new_embedding(vector_db="faiss") +``` + +### How can I set the collection store? +#### "I want to use a specific collection store" +You can set the collection store with the ``set_active_db`` method of the ``LLMWareConfig`` class. + +The collection store is set using the ``LLMWareConfig`` class with the ``set_active_db`` method. +At the time of writing, **LLMWare** supports the three collection stores *MongoDB*, *Postgres*, and *SQLite* - which is the default. +You can retrieve the supported collection store with the method ``get_supported_collection_db``. +In the example below, we first print the currently active collection store, then we retrieve the supported collection stores, before we switch to *Postgres*. + +```python +import logging + +from llmware.configs import LLMWareConfig + + +logging.info(f'Currently active collection store: {LLMWareConfig.get_active_db()}') +logging.info(f'Currently supported collection stores: {LLMWareConfig().get_supported_collection_db()}') + +LLMWareConfig.set_active_db("postgres") +logging.info(f'Currently active collection store: {LLMWareConfig.get_active_db()}') +``` + + +### How can I retrieve more context? +#### "I want to retrieve more context from a query" +One way to retrieve more context is to set the ``result_count`` parameter of the ``query``, ``text_query``, and ``semantic_query`` methods from the ``Query`` class. +By increasing ``result_count``, the number of retrieved results is increased which increases the context size. + +The ``Query`` class has the methods ``query``, ``text_query``, and ``semantic_query`` methods which allow to set the number of retrieved results with ``result_count``. +On a side note, ``query`` is a wrapper function for ``text_query`` and ``semantic_query``. +The value of ``result_count`` is passed on to the queried embedding store to control the number of retrieved results. +For example, for *pgvector* ``result_count`` is passed on to the value after the ``LIMIT`` keyword. +In the ``SQL`` example below, you can see the resulting ``SQL`` query of ``LLMWare`` if ``result_count=10``, the name of the collection being ``agreements``, and the query vector being ``[1, 2, 3]``. +```sql +SELECT + id, + block_mongo_id, + embedding <-> '[1, 2, 3]' AS distance, + text +FROM agreements +ORDER BY distance +LIMIT 10; +``` +In the following example, we execute the same query against a library twice but change the number of retrieved results from ``3`` to ``6``. +```python +import logging +from pathlib import Path + +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query + +logging.info(f'Currently supported embedding stores: {LLMWareConfig().get_supported_vector_db()}') + +library = Library().create_new_library(library_name='context_size_example') +library.add_files(input_foler_path=Path('~/llmware_data/sample_files/Agreements')) +library.install_new_embedding(vector_db="pg_vector") + +query = Query(library) +query_results = query.semantic_query(query='salary', result_count=3, results_only=True) +logging.info(f'Number of results: {len(query_results)}') +query_results = query.semantic_query(query='salary', result_count=6, results_only=True) +logging.info(f'Number of results: {len(query_results)}') +``` + +### How can I set the Large Language Model? +#### "I want to use a different LLM" +You can set the Large Language Model (LLM) with the ``gen_model`` parameter of the ``load_model`` method from the ``Prompt`` class. + +The ``Prompt`` class has the method ``load_model`` with the ``gen_model`` parameter which sets the LLM. +The ``gen_model`` parameter is passed on to the ``ModelCatalog`` class, which loads the LLM either from HuggingFace or from another source. +The ``ModelCatalog`` allows you to **list all available models** with the method ``list_generative_models``, or just the local models ``list_generative_local_models``, or just the open source models ``list_open_source_models``. +In the example below, we log all available LLMs, including the ones that are available locally and the open source ones, and also create the prompters. +Each prompter uses a different LLM from our [BLING model series](https://llmware.ai/about), which you can also find on [HuggingFace](https://huggingface.co/collections/llmware/bling-models-6553c718f51185088be4c91a). + +```python +import logging + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + + +llm_gen = ModelCatalog().list_generative_models() +logging.info(f'List of all LLMs: {llm_gen}') + +llm_gen_local = ModelCatalog().list_generative_local_models() +logging.info(f'List of all local LLMs: {llm_local}') + +llm_gen_open_source = ModelCatalog().list_open_source_models() +logging.info(f'List of all open source LLMs: {llm_gen_open_source}') + + +prompter_bling_1b = Prompt().load_model(gen_model='llmware/bling-1b-0.1') +prompter_bling_tiny_llama = Prompt().load_model(gen_model='llmware/bling-tiny-llama-v0') +prompter_bling_falcon_1b = Prompt().load_model(gen_model='llmware/bling-falcon-1b-0.1') +``` + +### How can I set the embedding model? +#### "I want to use a different embedding model" +You can set the embedding model with the ``embedding_model_name`` parameter of the ``install_new_embedding`` method from the ``Library`` class. + +The ``Library`` class has the method ``install_new_embedding`` with the ``embedding_model_name`` parameter which sets the embedding model. +The ``ModelCatalog`` allows you to **list all available embedding models** with the ``list_embedding_models`` method. +In the following example, we list all available embedding models, and then we create a library with the name ``embedding_models_example``, which we embed two times with embedding models ``'mini-lm-sber'`` and ``'industry-bert-contracts'``. + +```python +import logging + +from llmware.models import ModelCatalog +from llmware.library import Library + + +embedding_models = ModelCatalog().list_generative_models() +logging.info(f'List of embedding models: {embedding_models}') + + +library = Library().create_new_library(library_name='embedding_models_example') +library.add_files(input_foler_path=Path('~/llmware_data/sample_files/Agreements')) + +library.install_new_embedding(embedding_model_name='mini-lm-sber') +library.install_new_embedding(embedding_model_name='industry-bert-contracts') +``` + +### Why is the model running slowly in Google Colab? +#### "I want to improve the performance of my model on Google Colab" + +Our models are designed to run on at least 16GB of RAM. By default Google Colab provides ~13GB of RAM, which significantly slows computational speed. To ensure the best performance when using our models, we highly recommend enabling the T4 GPU in Colab. This will provide the notebook with additional resources, including 16GB of RAM, allowing our models to run smoothly and efficiently. + +Steps to enabling T4 GPU in Colab: +1. In your Colab notebook, click on the "Runtime" tab +2. Select "Change runtime type" +3. Under "Hardware Accelerator", select T4 GPU + +NOTE: There is a weekly usage limit on using T4 for free. \ No newline at end of file diff --git a/docs/community/join_our_community.md b/docs/community/join_our_community.md new file mode 100644 index 0000000..485f428 --- /dev/null +++ b/docs/community/join_our_community.md @@ -0,0 +1,71 @@ +--- +layout: default +title: Join Our Community +parent: Community +nav_order: 4 +description: overview of the major modules and classes of LLMWare +permalink: /community/join_our_community +--- +# Join the LLMWare Community +___ + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/community/need_help.md b/docs/community/need_help.md new file mode 100644 index 0000000..95c8a62 --- /dev/null +++ b/docs/community/need_help.md @@ -0,0 +1,71 @@ +--- +layout: default +title: Need Hep +parent: Community +nav_order: 3 +description: overview of the major modules and classes of LLMWare +permalink: /community/need_help +--- +# Need Help +___ + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/community/troubleshooting.md b/docs/community/troubleshooting.md new file mode 100644 index 0000000..3e703b2 --- /dev/null +++ b/docs/community/troubleshooting.md @@ -0,0 +1,166 @@ +--- +layout: default +title: Troubleshooting +parent: Community +nav_order: 2 +description: overview of the major modules and classes of LLMWare +permalink: /community/troubleshooting +--- +# Common Troubleshooting Issues +___ + + +1. **Can not install the pip package** + + -- Check your Python version. If using Python 3.9-3.11, then almost any version of llmware should work. If using an older Python (before 3.9), then it is likely that dependencies will fail in the pip process. If you are using Python 3.12, then you need to use llmware>=0.2.12. + + -- Dependency constraint error. If you receive a specific error around a dependency version constraint, then please raise an issue and include details about your OS, Python version, any unique elements in your virtual environment, and specific error. + + +2. **Parser module not found** + + -- Check your OS and confirm that you are using a [supported platform](platforms.md/#platform-support). + -- If you cloned the repository, please confirm that the /lib folder has been copied into your local path. + + +3. **Pytorch Model not loading** + + -- Confirm the obvious stuff - correct model name, model exists in Huggingface repository, connected to the Internet with open ports for HTTPS connection, etc. + + -- Check Pytorch version - update Pytorch to >2.0, which is required for many recent models released in the last 6 months, and in some cases, may require other dependencies not included in the llmware package. + --note: we have seen some compatibility issues with Pytorch==2.3 on Wintel platforms - if you run into these issues, we recommend using a back-level Pytorch==2.1, which we have seen fixing the issue. + +4. **GGUF Model not loading** + + -- Confirm that you are using llmware>=0.2.11 for the latest GGUF support. + + -- Confirm that you are using a [supported platform](platforms.md/#platform-support). We provide pre-built binaries for llama.cpp as a back-end GGUF engine on the following platforms: + + - Mac M1/M2/M3 - OS version 14 - "with accelerate framework" + - Mac M1/M2/M3 - OS older versions - "without accelerate framework" + - Windows - x86 + - Windows with CUDA + - Linux - x86 (Ubuntu 20+) + - Linux with CUDA (Ubuntu 20+) + +If you are using a different OS platform, you have the option to "bring your own llama.cpp" lib as follows: + +```python +from llmware.gguf_configs import GGUFConfigs +GGUFConfigs().set_config("custom_lib_path", "/path/to/your/libllama_binary") +``` + +If you have any trouble, feel free to raise an Issue and we can provide you with instructions and/or help compiling llama.cpp for your platform. + + -- Specific GGUF model - if you are successfully using other GGUF models, and only having problems with a specific model, then please raise an Issue, and share the specific model and architecture. + + +5. **Example not working as expected** - please raise an issue, so we can evaluate and fix any bugs in the example code. Also, pull requests are always especially welcomed with a fix or improvement in an example. + + +6. **Model not leveraging CUDA available in environment.** + + -- **Check CUDA drivers installed correctly** - easy check of the NVIDIA CUDA drivers is to use `nvidia-smi` and `nvcc --version` from the command line. Both commands should respond positively with details on the versions and implementations. Any errors indicates that either the driver or CUDA toolkit are not installed or recognized. It can be complicated at times to debug the environment, usually with some trial and error. See extensive [Nvidia Developer documentation](https://docs.nvidia.com) for trouble-shooting steps, specific to your environment. + + -- **Check CUDA drivers are up to date** - we build to CUDA 12.1, which translates to a minimum of 525.60 on Linux, and 528.33 on Windows. + + -- **Pytorch model** - check that Pytorch is finding CUDA, e.g., `torch.cuda.is_available()` == True. We have seen issues on Windows, in particular, to confirm that your Pytorch version has been compiled with CUDA drivers. For Windows, in particular, we have found that you may need to compile a CUDA-specific version of Pytorch, using the following command: + + ```pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121``` + + -- **GGUF model** - logs will be displayed on the screen confirming that CUDA is being used, or whether 'fall-back' to CPU drivers. We run a custom CUDA install check, which you can run on your system with: + ```gpu_status = ModelCatalog().gpu_available``` + + If you are confirming CUDA present, but fall-back to CPU is being used, you can set the GGUFConfigs to force to CUDA: + ```GGUFConfigs().set_config("force_gpu", True)``` + + If you are looking to use specific optimizations, you can bring your own llama.cpp lib as follows: + ```GGUFConfigs().set_config("custom_lib_path", "/path/to/your/custom/llama_cpp_backend")``` + + -- If you can not debug after these steps, then please raise an Issue. We are happy to dig in and work with you to run FAST local inference. + + +7. **Model result inconsistent** + + -- when loading the model, set `temperature=0.0` and `sample=False` -> this will give a deterministic output for better testing and debugging. + + -- usually the issue will be related to the retrieval step and formation of the Prompt, and as always, good pipelines and a little experimentation usually help ! + + +8. **Newly added examples not working as intended** + + -- If you run a recently added example and it does not run as intended, it is possible that the feature being used in the example has not yet been added to the latest pip install. + + -- To fix this, move the example file to the outer-most directory of the repository, so that the example file you are trying to run is in the same directory as the `llmware` source code directory. + + -- This will let you run the example using the latest source code! + + +9. **Git permission denied error** + + -- If you are using SSH to clone the repository and you get an error that looks similar to `git@github.com: Permission denied (publickey)`, then you might not have configured your SSH key correctly. + + -- If you don't already have one, you will need to create a new SSH key on your local machine. For instructions on how to do this, check out this page: https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent. + + -- You then need to add the SSH key to your GitHub account. For instructions on how to do this, check out this page: https://docs.github.com/en/authentication/connecting-to-github-with-ssh/adding-a-new-ssh-key-to-your-github-account. + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/components/agent_inference_server.md b/docs/components/agent_inference_server.md new file mode 100644 index 0000000..dcda509 --- /dev/null +++ b/docs/components/agent_inference_server.md @@ -0,0 +1,194 @@ +--- +layout: default +title: Agent Inference Server +parent: Components +nav_order: 12 +description: overview of the major modules and classes of LLMWare +permalink: /components/agent_inference_server +--- +# Agent Inference Server +--- + +LLMWare supports multiple deployment options, including the use of REST APIs to implement most model invocations. + +To set up an inference server for Agent processes: + +```python + +""" This example shows how to set up an inference server that can be used in conjunction with agent-based workflows. + + This script covers both the server-side deployment, as well as the steps taken on the client-side to deploy + in an Agent example. + + Note: this example will build off two other examples: + + 1. "examples/Models/launch_llmware_inference_server.py" + 2. "examples/SLIM-Agents/agent-llmfx-getting-started.py" + +""" + + +from llmware.models import ModelCatalog, LLMWareInferenceServer + +# *** SERVER SIDE SCRIPT *** + +base_model = "llmware/bling-tiny-llama-v0" +LLMWareInferenceServer(base_model, + model_catalog=ModelCatalog(), + secret_api_key="demo-test", + home_path="/home/ubuntu/", + verbose=True).start() + +# this will start Flask-based server, which will display the launched IP address and port, e.g., +# "Running on " ip_address = "http://127.0.0.1:8080" + + +# *** CLIENT SIDE AGENT PROCESS *** + + +from llmware.agents import LLMfx + + +def create_multistep_report_over_api_endpoint(): + + """ This is derived from the script in the example agent-llmfx-getting-started.py. """ + + customer_transcript = "My name is Michael Jones, and I am a long-time customer. " \ + "The Mixco product is not working currently, and it is having a negative impact " \ + "on my business, as we can not deliver our products while it is down. " \ + "This is the fourth time that I have called. My account number is 93203, and " \ + "my user name is mjones. Our company is based in Tampa, Florida." + + # create an agent using LLMfx class + agent = LLMfx() + + # copy the ip address from the Flask launch readout + ip_address = "http://127.0.0.1:8080" + + # inserting this line below into the agent process sets the 'api endpoint' execution to "ON" + # all agent function calls will be deployed over the API endpoint on the remote inference server + # to "switch back" to local execution, comment out this line + + agent.register_api_endpoint(api_endpoint=ip_address, + api_key="demo-test", + endpoint_on=True) + + # to explicitly turn the api endpoint "on" or "off" + # agent.switch_endpoint_on() + # agent.switch_endpoint_off() + + agent.load_work(customer_transcript) + + # load tools individually + agent.load_tool("sentiment") + agent.load_tool("ner") + + # load multiple tools + agent.load_tool_list(["emotions", "topics", "intent", "tags", "ratings", "answer"]) + + # start deploying tools and running various analytics + + # first conduct three 'soft skills' initial assessment using 3 different models + agent.sentiment() + agent.emotions() + agent.intent() + + # alternative way to execute a tool, passing the tool name as a string + agent.exec_function_call("ratings") + + # call multiple tools concurrently + agent.exec_multitool_function_call(["ner","topics","tags"]) + + # the 'answer' tool is a quantized question-answering model - ask an 'inline' question + # the optional 'key' assigns the output to a dictionary key for easy consolidation + agent.answer("What is a short summary?",key="summary") + + # prompting tool to ask a quick question as part of the analytics + response = agent.answer("What is the customer's account number and user name?", key="customer_info") + + # you can 'unload_tool' to release it from memory + agent.unload_tool("ner") + agent.unload_tool("topics") + + # at end of processing, show the report that was automatically aggregated by key + report = agent.show_report() + + # displays a summary of the activity in the process + activity_summary = agent.activity_summary() + + # list of the responses gathered + for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + + output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + + return output +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • + +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/agents.md b/docs/components/agents.md new file mode 100644 index 0000000..c14cd47 --- /dev/null +++ b/docs/components/agents.md @@ -0,0 +1,136 @@ +--- +layout: default +title: Agents +parent: Components +nav_order: 4 +description: overview of the major modules and classes of LLMWare +permalink: /components/agents +--- +# Agents +--- + +Agents with Function Calls and SLIM Models 🔥 + +llmware has been designed to enable Agent and LLM-based function calls using small language models designed for local and private +deployment and the ability to leverage open source models to conduct complex RAG and knowledge-based workflow automation. + +The key elements in llmware: + + - **SLIM models** - 18 function-calling small language models, optimized for a specific extraction, classification, generation, or +summarization activity, and generate python dictionaries and lists as output. + +- **LLMfx class** - enables a wide range of agent-based processes. + +Here is an example to get started: + +```python + +from llmware.agents import LLMfx + +text = ("Tesla stock fell 8% in premarket trading after reporting fourth-quarter revenue and profit that " + "missed analysts’ estimates. The electric vehicle company also warned that vehicle volume growth in " + "2024 'may be notably lower' than last year’s growth rate. Automotive revenue, meanwhile, increased " + "just 1% from a year earlier, partly because the EVs were selling for less than they had in the past. " + "Tesla implemented steep price cuts in the second half of the year around the world. In a Wednesday " + "presentation, the company warned investors that it’s 'currently between two major growth waves.'") + +# create an agent using LLMfx class +agent = LLMfx() + +# load text to process +agent.load_work(text) + +# load 'models' as 'tools' to be used in analysis process +agent.load_tool("sentiment") +agent.load_tool("extract") +agent.load_tool("topics") +agent.load_tool("boolean") + +# run function calls using different tools +agent.sentiment() +agent.topics() +agent.extract(params=["company"]) +agent.extract(params=["automotive revenue growth"]) +agent.xsum() +agent.boolean(params=["is 2024 growth expected to be strong? (explain)"]) + +# at end of processing, show the report that was automatically aggregated by key +report = agent.show_report() + +# displays a summary of the activity in the process +activity_summary = agent.activity_summary() + +# list of the responses gathered +for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + +output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/components.md b/docs/components/components.md new file mode 100644 index 0000000..16cbf10 --- /dev/null +++ b/docs/components/components.md @@ -0,0 +1,164 @@ +--- +layout: default +title: Components +nav_order: 3 +has_children: true +description: llmware key architectural components, modules and classes +permalink: /components +--- +# LLMWare Architecture +--- + +llmware is characterized by a logically integrated set of data pipelines involved in building LLM-based workflows, centered on two main sub-pipelines with high-level interfaces intended to provide an abstraction layer over individual 'end point' components to promote code re-use and the ability to easily 'swap' different components with minimal, if any, code change: + +**1. Knowledge Ingestion** - "creating Gen Ai Food" - ingesting and organizing unstructured information from a wide range of data sources, including each of the major steps: + + - Extracting and Parsing + - Text Chunking + - Indexing, Organizing and Storing + - Embedding + - Retrieval + - Analytics and Reuse of Content + - Combining with SQL Table and Other Structured Content + + + **Core LLMWare classes**: **Library**, **Query** (retrieval module), **Parser**, **EmbeddingHandler** (embeddings module), **Graph**, **CustomTables** (resources module) and **Datasets** dataset_tools module). + + In many cases, it is easy to get things done in LLMWare using only **Library** and **Query** - which provide convenient interfaces into parsing and embedding such that most use cases will not require calling those classes directly. + + Supported document file types: pdf, pptx, docx, xlsx, txt, csv, html, jsonl, json, tsv, jpg, jpeg, png, wav, zip, md, mp3, mp4, m4a + + Key methods to know: + + - Ingest anything - `Library().add_files(input_folder_path="path/to/docs")` + + - Embed library - `Library().install_new_embedding(embedding_model_name="your embedding model", vector_db="your vector db")` + + - Run Query - `Query(library).query(query, query_type="semantic", result_count=20)` + + Top examples to get started: + + - [Parsing examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing) - ~14 stand-alone parsing examples for all common document types, including options for parsing in memory, outputting to JSON, parsing custom configured CSV and JSON files, running OCR on embedded images found in documents, table extraction, image extraction, text chunking, zip files, and web sources. + - [Embedding examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Embedding) - ~15 stand-alone embedding examples to show how to use ~10 different vector databases and wide range of leading open source embedding models (including sentence transformers). + - [Retrieval examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Retrieval) - ~10 stand-alone examples illustrating different query and retrieval techniques - semantic queries, text queries, document filters, page filters, 'hybrid' queries, author search, using query state, and generating bibliographies. + - [Dataset examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Datasets) - ~5 stand-alone examples to show 'next steps' of how to leverage a Library to re-package content into various datasets and automated NLP analytics. + - [Fast start example #1-Parsing](https://www.github.com/llmware-ai/llmware/tree/main/fast_start/example-1-create_first_library.py) - shows the basics of parsing. + - [Fast start example #2-Embedding](https://www.github.com/llmware-ai/llmware/tree/main/fast_start/example-2-build_embeddings.py) - shows the basics of building embeddings. + - [CustomTable examples](https://www.github.com/llmware-ai/llmware/tree/main/Structured_Tables) - ~5 examples to start building structured tables that can be used in conjunction with LLM-based workflows. + + +**2. Model Prompting** - "Fun with LLMs" - the lifecycle of discovering, instantiating, and configuring an LLM-based model to execute an inference, including the ability to seamlessly prepare and integrate knowledge retrieval, and post-processing steps to validate accuracy, including: + + - ModelCatalog - discover, load and manage configuration + - Inference + - Function Calls + - Prompts + - Prompt with Sources + - Fact Checking methods + - Agent-based multi-step processes + - Prompt History + + Core LLMWare classes: **ModelCatalog** (models module), **Prompt**, **LLMfx** (agents module). + + Key methods to know: + + - Discover Models - `ModelCatalog().list_all_models()` + + - Load Model - `model = ModelCatalog().load_model(model_name)` + + - Inference - `response = model.inference(prompt, add_context=context)` + + - Prompt - wraps the model class to provide easy source/retrieval management + + - LLMfx - wraps the model class for function-calling SLIM models for agent processes + + While ~17 individual model classes are exposed in the models module, for most use cases, we recommend working through the higher-level interface of ModelCatalog, as it promotes code re-use and the easy ability to swap models. In many pipelines, even ModelCatalog is not required to be called directly, as the Prompt class (knowledge retrieval) and LLMfx (agents and function calls) class provide seamless workflow capabilities and are built on top of the ModelCatalog. + + Top examples to get started: + - [Models examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Models) - ~20 examples showing a wide range of different model inferences and use cases, including the ability to integrate Ollama models, OpenChat (e.g., LMStudio) models, using LLama-3 and Phi-3, bringing your own models into the ModelCatalog, and configuring sampling settings. + - [Prompts examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Prompts) - ~5 examples that illustrate how to use Prompt as an integrated workflow for integrating knowledge sources, managing prompt history, and applying fact-checking. + - [SLIM-Agents examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/SLIM-Agents) - ~20 examples showing how to build multi-model, multi-step Agent processes using locally-running SLIM function calling models. + - [Fast start example #3-Prompts and Models](https://www.github.com/llmware-ai/llmware/tree/main/fast_start/example-3-prompts_and_models.py) - getting started with model inference. + + +In addition, to support these two key pipelines, LLMWare has a set of supporting and enabling classes and methods, including: + + - resource module: CollectionRetrieval, CollectionWriter, PromptState, QueryState, and ParserState - provides an abstraction layer on top of underlying database repositories and separate state mechanisms for major classes. + - gguf_configs module: GGUFConfigs + - model_configs module: global_model_repo_catalog_list, global_model_finetuning_prompt_wrappers_lookup, global_default_prompt_catalog + - util module: Utilities + - setup module: Setup + - status module: Status + - exceptions module: LLMWare Exceptions + - web_services module: classes for Wikipedia, YFinance, and WebSite extraction + + +**End-to-End Use Cases** - we publish and maintain a number of end-to-end use cases in [examples/Use_Cases](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases) + + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/data_stores.md b/docs/components/data_stores.md new file mode 100644 index 0000000..75831e9 --- /dev/null +++ b/docs/components/data_stores.md @@ -0,0 +1,102 @@ +--- +layout: default +title: Data Stores +parent: Components +nav_order: 9 +description: overview of the major modules and classes of LLMWare +permalink: /components/data_stores +--- +# Data Stores +--- + +Simple-to-Scale Database Options - integrated data stores from laptop to parallelized cluster. + +```python + +from llmware.configs import LLMWareConfig + +# to set the collection database - mongo, sqlite, postgres +LLMWareConfig().set_active_db("mongo") + +# to set the vector database (or declare when installing) +# --options: milvus, pg_vector (postgres), redis, qdrant, faiss, pinecone, mongo atlas +LLMWareConfig().set_vector_db("milvus") + +# for fast start - no installations required +LLMWareConfig().set_active_db("sqlite") +LLMWareConfig().set_vector_db("chromadb") # try also faiss and lancedb + +# for single postgres deployment +LLMWareConfig().set_active_db("postgres") +LLMWareConfig().set_vector_db("postgres") + +# to install mongo, milvus, postgres - see the docker-compose scripts as well as examples + +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/embedding_models.md b/docs/components/embedding_models.md new file mode 100644 index 0000000..05d6954 --- /dev/null +++ b/docs/components/embedding_models.md @@ -0,0 +1,144 @@ +--- +layout: default +title: Embedding Models +parent: Components +nav_order: 6 +description: overview of the major modules and classes of LLMWare +permalink: /components/embedding_models +--- +# Embedding Models +--- + +llmware supports 30+ embedding models out of the box in the default ModelCatalog, with easy extensibility to add other +popular open source embedding models from HuggingFace or Sentence Transformers. + +To get a list of the currently supported embedding models: + +```python +from llmware.models import ModelCatalog +embedding_models = ModelCatalog().list_embedding_models() +for i, models in enumerate(embedding_models): + print(f"embedding models: {i} - {models}") +``` + +Supported popular models include: +- Sentence Transformers - `all-MiniLM-L6-v2`, `all-mpnet-base-v2` +- Jina AI - `jinaai/jina-embeddings-v2-base-en`, `jinaai/jina-embeddings-v2-small-en` +- Nomic - `nomic-ai/nomic-embed-text-v1` +- Industry BERT - `industry-bert-insurance`, `industry-bert-contracts`, `industry-bert-asset-management`, `industry-bert-sec`, `industry-bert-loans` +- OpenAI - `text-embedding-ada-002`, `text-embedding-3-small`, `text-embedding-3-large` + +We also support top embedding models from BAAI, thenlper, llmrails/ember, Google, and Cohere. We are constantly looking to add new innovative open source models to this list +so please let us know if you are looking for support for a specific embedding model, and usually within 1-2 days, we can test and add to the ModelCatalog. + +# Using an Embedding Model + +Embedding models in llmware can be installed directly by `ModelCatalog().load_model("model_name")`, but in most cases, +the name of the embedding model will be passed to the `install_new_embedding` handler in the Library class when creating a new +embedding. Once that is completed, the embedding model is captured in the Library metadata on the LibraryCard as part of the +embedding record for that library, and as a result, often times, does not need to be used explicitly again, e.g., + +```python + +from llmware.library import Library + +library = Library().create_new_library("my_library") + +# parses the content from the documents in the file path, text chunks and indexes in a text collection database +library.add_files(input_folder_path="/local/path/to/my_files", chunk_size=400, max_chunk_size=600, smart_chunking=1) + +# creates embeddings - and keeps synchronized records of which text chunks have been embedded to enable incremental use +library.install_new_embedding(embedding_model_name="jinaai/jina-embeddings-v2-small-en", + vector_db="milvus", + batch_size=100) +``` + +Once the embeddings are installed on the library, you can look up the embedding status to see the updated embeddings, and confirm that +the model has been correctly captured: + +```python + +from llmware.library import Library +library = Library().load_library("my_library") +embedding_record = library.get_embedding_status() +print("\nupdate: embedding record - ", embedding_record) +``` + +And then you can run semantic retrievals on the Library, using the Query class in the retrievals module, e.g.: + +```python +from llmware.library import Library +from llmware.retrieval import Query +library = Library().load_library("my_library") +# queries are constructed by creating a Query object, and passing a library as input +query_results = Query(library).semantic_query("my query", result_count=20) +for qr in query_results: + print("my query results: ", qr) +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discord channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/gguf.md b/docs/components/gguf.md new file mode 100644 index 0000000..a46b521 --- /dev/null +++ b/docs/components/gguf.md @@ -0,0 +1,186 @@ +--- +layout: default +title: GGUF +parent: Components +nav_order: 14 +description: overview of the major modules and classes of LLMWare +permalink: /components/gguf +--- +# GGUF +--- + +llmware packages its own build of the llama.cpp backend engine to enable running quantized models in GGUF format, which provides an +effective packaging to run small language models on both CPUs and GPUs, which fast loading and inference. + +The GGUF capability is implemented in the models.py module in the class `GGUFGenerativeModel` with an extensive set of interfaces and +configurations provided in the gguf_configs.py module (which for most users and use cases do not need to adjusted). + +To use a GGUF model is the same as using any other model in the ModelCatalog, e.g., + +```python +from llmware.models import ModelCatalog + +gguf_model = ModelCatalog().load_model("phi-3-gguf") +response = gguf_model.inference("What are the benefits of small specialized language models?") +print("response: ", response) +``` + +# GGUF Platform Support +Within the llmware library, we currently package 6 separate builds of the gguf llama.cpp engine for the following platforms: + +# Mac M1/M2/M3 + - with Accelerate: "libllama_mac_metal.dylib" + - without Accelerate: "libllama_mac_metal_no_acc.dylib" (note: if you have an old Mac OS installed, it may not have full Accelerate support) + - By default on Mac M1/M2/M3, it will attempt to use the Accelerate (faster) back-end, and if that fails, then it will automatically revert to the no-acc version + +# Windows + - CUDA version + - CPU version + - Will look for CUDA drivers, and if found, will try to use the CUDA build, but if that fails, then it will automatically revert to the CPU version. + +# Linux + - CUDA version + - CPU version + - Will look for CUDA drivers, and if found, will try to use the CUDA build, but if that fails, then it will automatically revert to the CPU version. + + +# Troubleshooting CUDA on Windows and Linux + +Requirement: Nvidia CUDA 12.1+ +-- how to check: `nvcc --version` and `nvidia-smi` - if not found, then drivers are either not installed or not in $PATH and need to be configured +-- if you have older drivers (e.g., v11), then you will need to update them. + +# Bring your own custom llama.cpp gguf backend + +If you have a unique system requirement, or are looking to optimize for a particular BLAS library with your own build, you can bring your own as follows: +if you have a unique system requirement, you can build llama_cpp from source, and apply custom build settings - or find in the community a prebuilt llama_cpp library that matches your platform. Happy to help if you share the requirements. + +```python +from llmware.gguf_configs import GGUFConfigs +GGUFConfigs().set_config("custom_lib_path", "/path/to/your/custom/llama_cpp_backend") + +# ... and then load and run the model as usual - the GGUF model class will look at this config and load the llama.cpp found at the custom lib path. +``` + +# Streaming GGUF + +```python + +""" This example illustrates how to use the stream method for GGUF models for fast streaming of inference, +especially for real-time chat interactions. + + Please note that the stream method has been implemented for GGUF models starting in llmware-0.2.13. This will be +any model with GGUFGenerativeModel class, and generally includes models with names that end in "gguf". + + See also the chat UI example in the UI examples folder. + + We would recommend using a chat optimized model, and have included a representative list below. +""" + + +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs + +# sets an absolute output maximum for the GGUF engine - normally set by default at 256 +GGUFConfigs().set_config("max_output_tokens", 1000) + +chat_models = ["phi-3-gguf", + "llama-2-7b-chat-gguf", + "llama-3-instruct-bartowski-gguf", + "openhermes-mistral-7b-gguf", + "zephyr-7b-gguf", + "tiny-llama-chat-gguf"] + +model_name = chat_models[0] + +# maximum output can be set optionally at any number up to the "max_output_tokens" set +model = ModelCatalog().load_model(model_name, max_output=500) + +text_out = "" + +token_count = 0 + +# prompt = "I am interested in gaining an understanding of the banking industry. What topics should I research?" +prompt = "What are the benefits of small specialized LLMs?" + +# since model.stream provides a generator, then use as follows to consume the generator + +for streamed_token in model.stream(prompt): + + text_out += streamed_token + if text_out.strip(): + print(streamed_token, end="") + + token_count += 1 + +# final output text and token count + +print("\n\n***total text out***: ", text_out) +print("\n***total tokens***: ", token_count) +``` + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/library.md b/docs/components/library.md new file mode 100644 index 0000000..02904eb --- /dev/null +++ b/docs/components/library.md @@ -0,0 +1,150 @@ +--- +layout: default +title: Library +parent: Components +nav_order: 7 +description: overview of the major modules and classes of LLMWare +permalink: /components/library +--- +# Library: ingest, organize and index a collection of knowledge at scale - Parse, Text Chunk and Embed. +--- + +Library is the main organizing construct for unstructured information in LLMWare. Users can create one large library with all types of different content, or +can create multiple libraries with each library comprising a specific logical collection of information on a +particular subject matter, project/case/deal, or even different accounts/users/departments. + +Each Library consists of the following components: + +1. Collection on a Database - this is the core of the Library, and is created through parsing of documents, which +are then automatically chunked and indexed in a text collection database. This is the basis for retrieval, +and the collection that will be used as the basis for tracking any number of vector embeddings that can be +attached to a library collection. + +2. File archives - found in the llmware_data path, within Accounts, there is a folder structure for each Library. +All file-based artifacts for the Library are organized in these folders, including copies of all files added in +the library (very useful for retrieval-based applications), images extracted and indexed from the source +documents, as well as derived artifacts such as nlp and knowledge graph and datasets. + +3. Library Catalog - each Library is registered in the LibraryCatalog table, with a unique library_card that has +the key attributes and statistics of the Library. + +When a Library object is passed to the Parser, the parser will automatically route all information into the +Library structure. + +The Library also exposes convenience methods to easily install embeddings on a library, including tracking of +incremental progress. + +To parse into a Library, there is the very useful convenience methods, "add_files" which will invoke the Parser, +collate and route the files within a selected folder path, check for duplicate files, execute the parsing, +text chunking and insertion into the database, and update all of the Library state automatically. + +Libraries are the main index constructs that are used in executing a Query. Pass the library object when +constructing the Query object, and then all retrievals (text, semantic and hybrid) will be executed against +the content in that Library only. + + +```python + +from llmware.library import Library + +# to parse and text chunk a set of documents (pdf, pptx, docx, xlsx, txt, csv, md, json/jsonl, wav, png, jpg, html) + +# step 1 - create a library, which is the 'knowledge-base container' construct +# - libraries have both text collection (DB) resources, and file resources (e.g., llmware_data/accounts/{library_name}) +# - embeddings and queries are run against a library + +lib = Library().create_new_library("my_library") + +# step 2 - add_files is the universal ingestion function - point it at a local file folder with mixed file types +# - files will be routed by file extension to the correct parser, parsed, text chunked and indexed in text collection DB + +lib.add_files("/folder/path/to/my/files") + +# to install an embedding on a library - pick an embedding model and vector_db +lib.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="milvus", batch_size=500) + +# to add a second embedding to the same library (mix-and-match models + vector db) +lib.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="chromadb", batch_size=100) + +# easy to create multiple libraries for different projects and groups + +finance_lib = Library().create_new_library("finance_q4_2023") +finance_lib.add_files("/finance_folder/") + +hr_lib = Library().create_new_library("hr_policies") +hr_lib.add_files("/hr_folder/") + +# pull library card with key metadata - documents, text chunks, images, tables, embedding record +lib_card = Library().get_library_card("my_library") + +# see all libraries +all_my_libs = Library().get_all_library_cards() + +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/model_catalog.md b/docs/components/model_catalog.md new file mode 100644 index 0000000..0982ae7 --- /dev/null +++ b/docs/components/model_catalog.md @@ -0,0 +1,263 @@ +--- +layout: default +title: Model Catalog +parent: Components +nav_order: 2 +description: overview of the major modules and classes of LLMWare +permalink: /components/model_catalog +--- +# Model Catalog: +Access all models the same way with easy lookup, regardless of underlying implementation. + +- 150+ Models in Catalog with 50+ RAG-optimized BLING, DRAGON and Industry BERT models +- 18 SLIM function-calling small language models for Agent use cases +- Full support for GGUF, HuggingFace, Sentence Transformers and major API-based models +- Easy to extend to add custom models - see examples + +Generally, all models can be identified using either the `model_name` or `display_name`, which provides some flexibility +to expose a more "UI friendly" name or an informal short-name for a commonly-used model. + +The default model list is implemented in the model_configs.py module, which is then generally accessed in the models.py module through +the `ModelCatalog` class, which also provides the ability to add models of various types, over-write by loading a custom model catalog from json file, and +other useful interfaces into the list of models. + +```python + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# all models accessed through the ModelCatalog +models = ModelCatalog().list_all_models() + +# to use any model in the ModelCatalog - "load_model" method and pass the model_name parameter +my_model = ModelCatalog().load_model("llmware/bling-phi-3-gguf") +output = my_model.inference("what is the future of AI?", add_context="Here is the article to read") + +# to integrate model into a Prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") +response = prompter.prompt_main("what is the future of AI?", context="Insert Sources of information") +``` + +# ADD a Custom GGUF to the ModelCatalog + +```python +import time +import re +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# Step 1 - register new gguf model - we will pick the popular LLama-2-13B-chat-GGUF + +ModelCatalog().register_gguf_model(model_name="TheBloke/Llama-2-13B-chat-GGUF-Q2", + gguf_model_repo="TheBloke/Llama-2-13B-chat-GGUF", + gguf_model_file_name="llama-2-13b-chat.Q2_K.gguf", + prompt_wrapper="my_version_inst") + +# Step 2- if the prompt_wrapper is a standard, e.g., Meta's , then no need to do anything else +# -- however, if the model uses a custom prompt wrapper, then we need to define that too +# -- in this case, we are going to create our "own version" of the Meta wrapper + +ModelCatalog().register_new_finetune_wrapper("my_version_inst", main_start="", llm_start="") + +# Once we have completed these two steps, we are done - and can begin to use the model like any other + +prompter = Prompt().load_model("TheBloke/Llama-2-13B-chat-GGUF-Q2") + +question_list = ["I am interested in gaining an understanding of the banking industry. What topics should I research?", + "What are some tips for creating a successful business plan?", + "What are the best books to read for a class on American literature?"] + + +for i, entry in enumerate(question_list): + + start_time = time.time() + print("\n") + print(f"query - {i + 1} - {entry}") + + response = prompter.prompt_main(entry) + + # Print results + time_taken = round(time.time() - start_time, 2) + llm_response = re.sub("[\n\n]", "\n", response['llm_response']) + print(f"llm_response - {i + 1} - {llm_response}") + print(f"time_taken - {i + 1} - {time_taken}") +``` + +# ADD an Ollama Model + +```python + +from llmware.models import ModelCatalog + +# Step 1 - register your Ollama models in llmware ModelCatalog +# -- these two lines will register: llama2 and mistral models +# -- note: assumes that you have previously cached and installed both of these models with ollama locally + +# register llama2 +ModelCatalog().register_ollama_model(model_name="llama2",model_type="chat",host="localhost",port=11434) + +# register mistral - note: if you are using ollama defaults, then OK to register with ollama model name only +ModelCatalog().register_ollama_model(model_name="mistral") + +# optional - confirm that model was registered +my_new_model_card = ModelCatalog().lookup_model_card("llama2") +print("\nupdate: confirming - new ollama model card - ", my_new_model_card) + +# Step 2 - start using the Ollama model like any other model in llmware + +print("\nupdate: calling ollama llama 2 model ...") + +model = ModelCatalog().load_model("llama2") +response = model.inference("why is the sky blue?") + +print("update: example #1 - ollama llama 2 response - ", response) + +# Tip: if you are loading 'llama2' chat model from Ollama, note that it is already included in +# the llmware model catalog under a different name, "TheBloke/Llama-2-7B-Chat-GGUF" +# the llmware model name maps to the original HuggingFace repository, and is a nod to "TheBloke" who has +# led the popularization of GGUF - and is responsible for creating most of the GGUF model versions. +# --llmware uses the "Q4_K_M" model by default, while Ollama generally prefers "Q4_0" + +print("\nupdate: calling Llama-2-7B-Chat-GGUF in llmware catalog ...") + +model = ModelCatalog().load_model("TheBloke/Llama-2-7B-Chat-GGUF") +response = model.inference("why is the sky blue?") + +print("update: example #1 - [compare] - llmware / Llama-2-7B-Chat-GGUF response - ", response) + +# Now, let's try the Ollama Mistral model with a context passage + +model2 = ModelCatalog().load_model("mistral") + +context_passage= ("NASA’s rover Perseverance has gathered data confirming the existence of ancient lake " + "sediments deposited by water that once filled a giant basin on Mars called Jerezo Crater, " + "according to a study published on Friday. The findings from ground-penetrating radar " + "observations conducted by the robotic rover substantiate previous orbital imagery and " + "other data leading scientists to theorize that portions of Mars were once covered in water " + "and may have harbored microbial life. The research, led by teams from the University of " + "California at Los Angeles (UCLA) and the University of Oslo, was published in the " + "journal Science Advances. It was based on subsurface scans taken by the car-sized, six-wheeled " + "rover over several months of 2022 as it made its way across the Martian surface from the " + "crater floor onto an adjacent expanse of braided, sedimentary-like features resembling, " + "from orbit, the river deltas found on Earth.") + +response = model2.inference("What are the top 3 points?", add_context=context_passage) + +print("\nupdate: calling ollama mistral model ...") + +print("update: example #2 - ollama mistral response - ", response) + +# Step 3 - using the ollama discovery API - optional + +discovery = model2.discover_models() +print("\nupdate: example #3 - checking ollama model manifest list: ", discovery) + +if len(discovery) > 0: + # note: assumes tht you have at least one model registered in ollama -otherwise, may throw error + for i, models in enumerate(discovery["models"]): + print("ollama models: ", i, models) +``` + +# Add a LM Studio Model + +```python +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + + +# one step process: add the open chat model to the Model Registry +# key params: +# model_name = "my_open_chat_model1" +# api_base = uri_path to the proposed endpoint +# prompt_wrapper = alpaca | | chat_ml | hf_chat | human_bot +# -> Llama2-Chat +# hf_chat -> Zephyr-Mistral +# chat_ml -> OpenHermes - Mistral +# human_bot -> Dragon models +# model_type = "chat" (alternative: "completion") + +ModelCatalog().register_open_chat_model("my_open_chat_model1", + api_base="http://localhost:1234/v1", + prompt_wrapper="", + model_type="chat") + +# once registered, you can invoke like any other model in llmware + +prompter = Prompt().load_model("my_open_chat_model1") +response = prompter.prompt_main("What is the future of AI?") + + +# you can (optionally) register multiple open chat models with different api_base and model attributes + +ModelCatalog().register_open_chat_model("my_open_chat_model2", + api_base="http://localhost:5678/v1", + prompt_wrapper="hf_chat", + model_type="chat") +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/prompt_with_sources.md b/docs/components/prompt_with_sources.md new file mode 100644 index 0000000..3a27e20 --- /dev/null +++ b/docs/components/prompt_with_sources.md @@ -0,0 +1,186 @@ +--- +layout: default +title: Prompt with Sources +parent: Components +nav_order: 10 +description: overview of the major modules and classes of LLMWare +permalink: /components/prompt_with_sources +--- +# Prompt with Sources +--- +Prompt with Sources: the easiest way to combine knowledge retrieval with a LLM inference, and provides several high-level useful methods to +easily integrate a retrieval/query/parsing step into a prompt to be used as a source for running an inference on a model. + +This is best illustrated with a simple example: + +```python + +from llmware.prompts import Prompt + +# build a prompt and attach a model +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") + +# add_source_document method: accepts any supported document type, parses the file, and creates text chunks +# if a query is passed, then it will run a quick in-memory filtering search against the text chunks +# the text chunks are packaged into sources with all of the accompanying metadata from the file, and made +# available automatically in batches to be used in prompting - + +source = prompter.add_source_document("/folder/to/one/doc/", "filename", query="fast query") + +# to run inference with 'prompt with sources' -> source will be automatically added to the prompt +responses = prompter.prompt_with_source("my query") + +# depending upon the size of the source (and batching relative to the model context window, there may be more than +# a single inference run, so unpack potentially multiple responses + +for i, response in enumerate(responses): + print("response: ", i, response) +``` + +# FACT CHECKING + +Using prompt_with_source also provides integrated fact-checking methods that use the packaged source information to validate key +elements from the llm_response + +```python +from llmware.prompts import Prompt + +prompter = Prompt().load_model("bling-answer-tool", temperature=0.0, sample=False) + +# contract is parsed, text-chunked, and then filtered by "base salary' +source = prompter.add_source_document("/local/folder/path", "my_document.pdf", query="exact filter query") + +# calling the LLM with 'source' information from the contract automatically packaged into the prompt +responses = prompter.prompt_with_source("my question to the document", prompt_name="default_with_context") + +# run several fact checks + +# checks for numbers match +ev_numbers = prompter.evidence_check_numbers(responses) + +# looks for statistical overlap to identify potential sources for the llm response +ev_sources = prompter.evidence_check_sources(responses) + +# builds set of comparison stats between the llm_response and the sources +ev_stats = prompter.evidence_comparison_stats(responses) + +# identifies if a response is a "not found" response +z = prompter.classify_not_found_response(responses, parse_response=True, evidence_match=True,ask_the_model=False) + +for r, response in enumerate(responses): + print("LLM Response: ", response["llm_response"]) + print("Numbers: ", ev_numbers[r]["fact_check"]) + print("Sources: ", ev_sources[r]["source_review"]) + print("Stats: ", ev_stats[r]["comparison_stats"]) + print("Not Found Check: ", z[r]) +``` + +In addition to `add_source_document`, the Prompt class implements the following other methods to easily integrate sources into prompts: + +# Add Source - Query Results - Two Options + +```python + +from llmware.prompts import Prompt +from llmware.retrieval import Query +from llmware.library import Library + +# build a prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") + +# Option A - run query and then add query_results to the prompt +my_lib = Library().load_library("my_library") +results = Query(my_lib).query("my query") + +source2 = prompter.add_source_query_results(results) + +# Option B - run a new query against a library and load directly into a prompt +source3 = prompter.add_source_new_query(my_lib, query="my new query", query_type="semantic", result_count=15) + +``` + +# Add Other Sources + +```python + +from llmware.prompts import Prompt + +# build a prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") + +# add wikipedia articles as a source +wiki_source = prompter.add_source_wikipedia("topic", article_count=5, query="filter among retrieved articles") + +# add a website as a source +website_source = prompter.add_source_website("my_url", query="filter among website") + +# add an entire library (should be small, e.g., just a couple of documents) +source = prompter.add_source_library("my_library") + +``` + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/query.md b/docs/components/query.md new file mode 100644 index 0000000..81f4b91 --- /dev/null +++ b/docs/components/query.md @@ -0,0 +1,115 @@ +--- +layout: default +title: Query +parent: Components +nav_order: 8 +description: overview of the major modules and classes of LLMWare +permalink: /components/query +--- +# Retrieval & Query +--- + +Query libraries with mix of text, semantic, hybrid, metadata, and custom filters. The retrieval.py module implements the +`Query` class, which is the primary way that search and retrieval is performed. Each `Query` object, when constructed, +requires that a Library is passed as a mandatory parameter in the constructor. The Query object will operate against that +Library, and have access to all of Library's specific attributes, metadata and methods. + +Retrievals in llmware leverage the Library abstraction as the primary unit against which a particular query or retrieval is +executed. This provides the ability to have multiple distinct knowledge-bases, potentially aligned to different use cases, and/or +users, accounts and permissions. + +# Executing Queries + +```python +from llmware.retrieval import Query +from llmware.library import Library + +# step 1 - load a previously created library +lib = Library().load_library("my_library") + +# step 2 - create a query object +q = Query(lib) + +# step 3 - run lots of different queries (many other options in the examples) + +# basic text query +results1 = q.text_query("text query", result_count=20, exact_mode=False) + +# semantic query +results2 = q.semantic_query("semantic query", result_count=10) + +# combining a text query restricted to only certain documents in the library and "exact" match to the query +results3 = q.text_query_with_document_filter("new query", {"file_name": "selected file name"}, exact_mode=True) + +# to apply a specific embedding (if multiple on library), pass the names when creating the query object +q2 = Query(lib, embedding_model_name="mini_lm_sbert", vector_db="milvus") +results4 = q2.semantic_query("new semantic query") +``` + + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/rag_optimized_models.md b/docs/components/rag_optimized_models.md new file mode 100644 index 0000000..d20c64e --- /dev/null +++ b/docs/components/rag_optimized_models.md @@ -0,0 +1,326 @@ +--- +layout: default +title: RAG Optimized Models +parent: Components +nav_order: 3 +description: overview of the major modules and classes of LLMWare +permalink: /components/rag_optimized_models +--- +# RAG Optimized Models +--- + +RAG-Optimized Models - 1-7B parameter models designed for RAG workflow integration and running locally. + +## Meet our Models + +- **SLIM model series:** small, specialized models fine-tuned for function calling and multi-step, multi-model Agent workflows. +- **DRAGON model series:** Production-grade RAG-optimized 6-7B parameter models - "Delivering RAG on ..." the leading foundation base models. +- **BLING model series:** Small CPU-based RAG-optimized, instruct-following 1B-3B parameter models. +- **Industry BERT models:** out-of-the-box custom trained sentence transformer embedding models fine-tuned for the following industries: Insurance, Contracts, Asset Management, SEC. +- **GGUF Quantization:** we provide 'gguf' and 'tool' versions of many SLIM, DRAGON and BLING models, optimized for CPU deployment. + + + +```python +""" This 'Hello World' example demonstrates how to get started using local BLING models with provided context, using both +Pytorch and GGUF versions. """ + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +# this is the main script to be run + +def bling_meets_llmware_hello_world (model_name): + + t0 = time.time() + + # load the questions + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + # load the model + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"] + , prompt_name="default_with_context",temperature=0.30) + + # print out the results + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # list of 'rag-instruct' laptop-ready small bling models on HuggingFace + + pytorch_models = ["llmware/bling-1b-0.1", # most popular + "llmware/bling-tiny-llama-v0", # fastest + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3" # most accurate (and newest) + ] + + # Quantized GGUF versions generally load faster and run nicely on a laptop with at least 16 GB of RAM + gguf_models = ["bling-phi-3-gguf", "bling-stablelm-3b-tool", "dragon-llama-answer-tool", "dragon-yi-answer-tool", "dragon-mistral-answer-tool"] + + # try model from either pytorch or gguf model list + # the newest (and most accurate) is 'bling-phi-3-gguf' + + bling_meets_llmware_hello_world(gguf_models[0]) + + # check out the model card on Huggingface for RAG benchmark test performance results and other useful information +``` + + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/release_history.md b/docs/components/release_history.md new file mode 100644 index 0000000..fa7bb33 --- /dev/null +++ b/docs/components/release_history.md @@ -0,0 +1,153 @@ +--- +layout: default +title: Release History +parent: Components +nav_order: 15 +description: llmware is an integrated framework with over 50+ models for quickly developing LLM-based applications including Retrieval Augmented Generation (RAG) and Multi-Step Orchestration of Agent Workflows. +permalink: /components/release_history +--- +Release History +--- + +- For Specific Wheels: [Wheel Archives](https://www.github.com/llmware-ai/llmware/tree/main/wheel_archives) +- For Features Details: [Main README-'Release notes and Change Log'](https://www.github.com/llmware-ai/llmware/tree/main/) + +New wheels are built generally on PyPy on a weekly basis and updated on PyPy versioning. The development repo is updated +and current at all times, but may have updates that are not yet in the PyPy wheel. + +All wheels are built and tested on: + +1. Mac Metal +2. Windows x86 (+ with CUDA) +3. Linux x86 (+ with CUDA) - most testing on Ubuntu 22 and Ubuntu 20 - which are recommended. +4. Mac x86 (see 0.2.11 note below) +5. Linux aarch64* (see 0.2.7 note below) + +**Release Notes** + +--**0.3.0** released in the week of June 4, 2024 - continued pruning of required dependencies with split of python dependencies into a small minimal set of requirements (~10 in requirements.txt) that are included in the pip install, with an additional set of optional dependencies provided as 'extras', reflected in both the requirements_extras.txt file, and available over pip with the added instruction - `pip3 install 'llmware[full]'`. Notably, commonly used libraries such as transformers, torch and openai are now in the 'extras' as most llmware use cases do not require them, and this greatly simplifies the ability to install llmware. The `welcome_to_llmware.sh` and `welcome_to_llmware_windows.sh` have also been updated to install both the 'core' and 'extra' set of requirements. Other subtle, but significant, architectural changes include offering more extensibility for adding new model classes, configurable global base model methods for post_init and register, a new InferenceHistory state manager, and enhanced logging options. + +--**0.2.15** released in the week of May 20, 2024 - removed pytorch dependency as a global import, and shifted to dynamically loading of torch in the event that it is called in a specific model class. This enables running most of llmware code and examples without pytorch or transformers loaded. The main areas of torch (and transformers) dependency is in using HFGenerativeModels and HFEmbeddingModels. + + - note: we have seen some new errors caused with Pytorch 2.3 - which are resolved by down-leveling to `pip3 install torch==2.1` + - note: there are a couple of new warnings from within transformers and huggingface_hub libraries - these can be safely ignored. We have seen that keeping `local_dir_use_symlinks = False` when pulling model artifacts from Huggingface is still the safer option in some environments. + +--**0.2.13** released in the week of May 12, 2024 - clean up of dependencies in both requirements.txt and Setup (PyPi) - install of vector db python sdk (e.g., pymilvus, chromadb, etc) is now required as a separate step outside of the pip3 install llmware - attempt to keep dependency matrix as simple as possible and avoid potential dependency conflicts on install, especially for packages which in turn have a large number of dependencies. If you run into any issues with install dependencies, please raise an issue. + + +--**0.2.12** released in the week of May 5, 2024 - added Python 3.12 support, and deprecated the use of faiss for v3.12+. We have changed the "Fast Start" no-install option to use chromadb or lancedb rather than faiss. Refactoring of code especially with Datasets, Graph and Web Services as separate modules. + +--**0.2.11** released in the week of April 29, 2024 - updated GGUF libs for Phi-3 and Llama-3 support, and added new prebuilt shared libraries to support WhisperCPP. We are also deprecating support for Mac x86 going forward - will continue to support on most major components but not all new features going forward will be built specifically for Mac x86 (which Apple stopped shipping in 2022). Our intent is to keep narrowing our testing matrix to provide better support on key platforms. We have also added better safety checks for older versions of Mac OS running on M1/M2/M3 (no_acc option in GGUF and Whisper libs), as well as a custom check to find CUDA drivers on Windows (independent of Pytorch). + +--**0.2.9** released in the week of April 15, 2024 - minor continued improvements to the parsers plus roll-out of new CustomTable class for rapidly integrating structured information into LLM-based workflows and data pipelines, including converting JSON/JSONL files and CSV files into structured DB tables. + +--**0.2.8** released in the week of April 8, 2024 - significant improvements to the Office parser with new libs on all platforms. Conforming changes with the PDF parser in terms of exposing more options for text chunking strategies, encoding, and range of capture options (e.g., tables, images, header text, etc). Linux aarch64 libs deprecated and kept at 0.2.6 - some new features will not be available on Linux aarch64 - we recommend using Ubuntu20+ on x86_64 (with and without CUDA). + +--**0.2.7** released in the week of April 1, 2024 - significant improvements to the PDF parser with new libs on all platforms. Important note that we are keeping linux aarch64 at 0.2.6 libs - and will be deprecating support going forward. For Linux, we recommend Ubuntu20+ and x86_64 (with and without CUDA). + +--**0.2.5** released in the week of March 12, 2024 - continued enhancements of the GGUF implementation, especially for CUDA support, and re-compiling of all binaries to support Ubuntu 20 and Ubuntu 22. Ubuntu requirements are: CUDA 12.1 (to use GPU), and GLIBC 2.31+. + +--**GGUF on Windows CUDA**: useful notes and debugging tips - + + 1. Requirement: Nvidia CUDA 12.1+ + + -- how to check: `nvcc --version` and `nvidia-smi` - if not found, then drivers are either not installed or not in $PATH and need to be configured + -- if you have older drivers (e.g., v11), then you will need to update them. + + 2. Requirement: CUDA-enabled Pytorch (pre-0.2.11) + + -- starting with 0.2.11, we have implemented a custom check to evaluate if CUDA is present, independent of Pytorch. + -- for pre-0.2.11, we use Pytorch to check for CUDA drivers, e.g., `torch.cuda.is_available()` and `torch.version.cuda` + + 3. Installing a CUDA-enabled Pytorch - useful install script: (not required post-0.2.11 for GGUF on Windows) + + -- `pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121` + + 4. Fall-back to CPU - if llmware can not load the CUDA-enabled drivers, it will automatically try to fall back to the CPU version of the drivers. + + -- you can also adjust the GGUFConfigs().set_config - ("use_gpu", False) - and then it will automatically go to the CPU drivers. + + 5. Custom GGUF libraries - if you have a unique system requirement, you can build llama_cpp from source, and apply custom build settings - or find in the community a prebuilt llama_cpp library that matches your platform. Happy to help if you share the requirements. + + -- to "bring your own GGUF": GGUFConfigs().set_config("custom_lib_path", "/path/to/your/custom/llama_cpp_backend" -> and llmware will try to load that library. + + 6. Issues? - please raise an Issue on Github, or on Discord - and we can work with you to get you up and running! + +--**0.2.4** released in the week of February 26, 2024 - major upgrade of GGUF implementation to support more options, including CUDA support - which is the main source of growth in the size of the wheel package. + + -- Note: We will look at making some of the CUDA builds as 'optional' or 'bring your own' over time. + -- Note: We will also start to 'prune' the list of wheels kept in the archive to keep the total repo size manageable for cloning. + +--**0.2.2** introduced SLIM models and the new LLMfx class, and the capabilities for multi-model, multi-step Agent-based processes. + +--**0.2.0** released in the week of January 22, 2024 - significant enhancements, including integration of Postgres and SQLite drivers into the c lib parsers. + +--New examples involving Postgres or SQLite support (including 'Fast Start' examples) will require a fresh pip install of 0.2.0 or clone of the repo. + +--If cloning the repo, please be especially careful to pick up the new updated /lib dependencies for your platform. + +--New libs have new dependencies in Linux in particular - most extensive testing on Ubuntu 22. If any issues on a specific version of Linux, please raise a ticket. + + + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/components/slim_models.md b/docs/components/slim_models.md new file mode 100644 index 0000000..ed1ac14 --- /dev/null +++ b/docs/components/slim_models.md @@ -0,0 +1,304 @@ +--- +layout: default +title: SLIM Models +parent: Components +nav_order: 5 +description: overview of the major modules and classes of LLMWare +permalink: /components/slim_models +--- +# SLIM Models - Function Calling with Small Language Models +--- + +Generally, function-calling is a specialized capability of frontier language models, such as OpenAI GPT4. + +We have adapted this concept to small language models through SLIMs (Structured Language Instruction Models), +which are 'single function' models fine-tuned to accept three main inputs to construct a prompt: + +As of June 2024, there are 18 distinct SLIM function calling models with many more on the way, for most common +extraction, classification, and summarization tasks: + +**Models List** +If you would like more information about any of the SLIM models, please check out their model card: + +- extract - extract custom keys - [slim-extract](https://www.huggingface.co/llmware/slim-extract) & [slim-extract-tool](https://www.huggingface.co/llmware/slim-extract-tool) +- summary - summarize function call - [slim-summary](https://www.huggingface.co/llmware/slim-summary) & [slim-summary-tool](https://www.huggingface.co/llmware/slim-summary-tool) +- xsum - title/headline function call - [slim-xsum](https://www.huggingface.co/llmware/slim-xsum) & [slim-xsum-tool](https://www.huggingface.co/llmware/slim-xsum-tool) +- ner - extract named entities - [slim-ner](https://www.huggingface.co/llmware/slim-ner) & [slim-ner-tool](https://www.huggingface.co/llmware/slim-ner-tool) +- sentiment - evaluate sentiment - [slim-sentiment](https://www.huggingface.co/slim-sentiment) & [slim-sentiment-tool](https://www.huggingface.co/llmware/slim-sentiment-tool) +- topics - generate topic - [slim-topics](https://www.huggingface.co/slim-topics) & [slim-topics-tool](https://www.huggingface.co/llmware/slim-topics-tool) +- sa-ner - combo model (sentiment + named entities) - [slim-sa-ner](https://www.huggingface.co/slim-sa-ner) & [slim-sa-ner-tool](https://www.huggingface.co/llmware/slim-sa-ner-tool) +- boolean - provides a yes/no output with explanation - [slim-boolean](https://www.huggingface.co/slim-boolean) & [slim-boolean-tool](https://www.huggingface.com/llmware/slim-boolean-tool) +- ratings - apply 1 (low) - 5 (high) rating - [slim-ratings](https://www.huggingface.co/slim-ratings) & [slim-ratings-tool](https://www.huggingface.co/llmware/slim-ratings-tool) +- emotions - assess emotions - [slim-emotions](https://www.huggingface.co/slim-emotions) & [slim-emotions-tool](https://www.huggingface.co/llmware/slim-emotions-tool) +- tags - auto-generate list of tags - [slim-tags](https://www.huggingface.co/slim-tags) & [slim-tags-tool](https://www.huggingface.co/llmware/slim-tags-tool) +- tags-3b - enhanced auto-generation tagging model - [slim-tags-3b](https://www.huggingface.com/slim-tags-3b) & [slim-tags-3b-tool](https://www.huggingface.co/llmware/slim-tags-3b-tool) +- intent - identify intent - [slim-intent](https://www.huggingface.co/slim-intent) & [slim-intent-tool](https://www.huggingface.co/llmware/slim-intent-tool) +- category - high-level category - [slim-category](https://www.huggingface.co/slim-category) & [slim-category-tool](https://wwww.huggingface.co/llmware/slim-category-tool) +- nli - assess if evidence supports conclusion - [slim-nli](https://www.huggingface.co/slim-nli) & [slim-nli-tool](https://www.huggingface.co/llmware/slim-nli-tool) +- sql - convert text into sql - [slim-sql](https://www.huggingface.co/slim-sql) & [slim-sql-tool](https://www.huggingface.co/llmware/slim-sql-tool) + +You may also want to check out these quantized 'answer' tools, which work well in conjunction with SLIMs for question-answer and summarization: +- bling-stablelm-3b-tool - 3b quantized RAG model - [bling-stablelm-3b-gguf](https://www.huggingface.co/llmware/bling-stablelm-3b-gguf) +- bling-answer-tool - 1b quantized RAG model - [bling-answer-tool](https://www.huggingface.co/llmware/bling-answer-tool) +- dragon-yi-answer-tool - 6b quantized RAG model - [dragon-yi-answer-tool](https://www.huggingface.co/llmware/dragon-yi-answer-tool) +- dragon-mistral-answer-tool - 7b quantized RAG model - [dragon-mistral-answer-tool](https://www.huggingface.co/llmware/dragon-mistral-answer-tool) +- dragon-llama-answer-tool - 7b quantized RAG model - [dragon-llama-answer-tool](https://www.huggingface.co/llmware/dragon-llama-answer-tool) + +All SLIM models have a common prompting structure + +Inputs: + -- text passage - this is the core passage or piece of text that you would like the model to assess + -- function - classify, extract, generate - this is handled by default by the model class, so usually does + not need to be explicitly declared - but is an option for SLIMs that support more than one function + -- params - depends upon the model, used to configure/guide the behavior of the function call - optional for + some SLIMs + +Outputs: + -- structured python output, generally either a dictionary or list + +Main objectives: + -- enable function calling with small, locally-running models, + -- simplify prompts by defining specific functions and fine-tuning the model to respond accordingly + without 'prompt magic' + -- standardized outputs that can be handled programmatically as part of a multi-step workflow. + + +```python + + +from llmware.models import ModelCatalog + + +def discover_slim_models(): + + """ Discover a list of SLIM tools in the Model Catalog. + + -- SLIMs are available in both traditional Pytorch and quantized GGUF packages. + -- Generally, we train/fine-tune in Pytorch and then package in 4-bit quantized GGUF for inference. + -- By default, we designate the GGUF versions with 'tool' or 'gguf' in their names. + -- GGUF versions are generally faster to load, faster for inference and use less memory in most environments.""" + + tools = ModelCatalog().list_llm_tools() + tool_map = ModelCatalog().get_llm_fx_mapping() + + print("\nList of SLIM model tools (GGUF) in the ModelCatalog\n") + + for i, tool in enumerate(tools): + model_card = ModelCatalog().lookup_model_card(tool_map[tool]) + print(f"{i} - tool: {tool} - " + f"model_name: {model_card['model_name']} - " + f"model_family: {model_card['model_family']}") + + return 0 + + +def hello_world_slim(): + + """ SLIM models can be identified in the ModelCatalog like any llmware model. Instead of using + inference method, SLIM models are used with the function_call method that prepares a special prompt + instruction, and takes optional parameters. + + This example shows a series of function calls with different SLIM models. + + Please note that the first time the models will be pulled from the llmware Huggingface repository, and will + take a couple of minutes. Future calls will be much faster once cached in memory locally. """ + + print("\nExecuting Function Call Inferences with SLIMs\n") + + # Sentiment Analysis + + passage1 = ("This is one of the best quarters we can remember for the industrial sector " + "with significant growth across the board in new order volume, as well as price " + "increases in excess of inflation. We continue to see very strong demand, especially " + "in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would " + "be accumulating more stock on any dips.") + + # here are the two key lines of code + model = ModelCatalog().load_model("slim-sentiment-tool") + response = model.function_call(passage1) + + print("sentiment response: ", response['llm_response']) + + # Named Entity Recognition + + passage2 = "Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s." + + model = ModelCatalog().load_model("slim-ner-tool") + response = model.function_call(passage2) + + print("ner response: ", response['llm_response']) + + # Extract anything with Slim-extract + + passage3 = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + + model = ModelCatalog().load_model("slim-extract-tool") + response = model.function_call(passage3, function="extract", params=["revenue growth"]) + + print("extract response: ", response['llm_response']) + + # Generate questions with Slim-Q-Gen + + model = ModelCatalog().load_model("slim-q-gen-tiny-tool", temperature=0.2, sample=True) + # supported params - "question", "multiple choice", "boolean" + response = model.function_call(passage3, params=['multiple choice']) + + print("question generation response: ", response['llm_response']) + + # Generate topic + + model = ModelCatalog().load_model("slim-topics-tool") + response = model.function_call(passage3) + + print("topics response: ", response['llm_response']) + + # Generate headline summary with slim-xsum + model = ModelCatalog().load_model("slim-xsum-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("xsum response: ", response['llm_response']) + + # Generate boolean with optional '(explain)` in parameter + model = ModelCatalog().load_model("slim-boolean-tool") + response = model.function_call(passage3, params=["Did Adobe revenue increase? (explain)"]) + + print("boolean response: ", response['llm_response']) + + # Generate tags + model = ModelCatalog().load_model("slim-tags-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("tags response: ", response['llm_response']) + + return 0 + + +def using_logits_and_integrating_into_process(): + + """ This example shows two key elements of function calling SLIM models - + + 1. Using Logit Information to indicate confidence levels, especially for classifications. + 2. Using the structured dictionary generated for programmatic handling in a larger process. + + """ + + print("\nExample: using logits and integrating into process\n") + + text_passage = ("On balance, this was an average result, with earnings in line with expectations and " + "no big surprises to either the positive or the negative.") + + # two key lines (load_model + execute function_call) + additional logit_analysis step + sentiment_model = ModelCatalog().load_model("slim-sentiment-tool", get_logits=True) + response = sentiment_model.function_call(text_passage) + analysis = ModelCatalog().logit_analysis(response,sentiment_model.model_card, sentiment_model.hf_tokenizer_name) + + print("sentiment response: ", response['llm_response']) + + print("\nAnalyzing response") + for keys, values in analysis.items(): + print(f"{keys} - {values}") + + # two key attributes of the sentiment output dictionary + sentiment_value = response["llm_response"]["sentiment"] + confidence_level = analysis["confidence_score"] + + # use the sentiment classification as a 'if...then' decision point in a process + if "positive" in sentiment_value: + print("sentiment is positive .... will take 'positive' analysis path ...", sentiment_value) + else: + print("sentiment is negative .... will take 'negative' analysis path ...", sentiment_value) + + if "positive" in sentiment_value and confidence_level > 0.8: + print("sentiment is positive with high confidence ... ", sentiment_value, confidence_level) + + return 0 + + +if __name__ == "__main__": + + # discovering slim models in the llmware catalog + discover_slim_models() + + # running function call inferences + hello_world_slim() + + # doing interesting stuff with the output + using_logits_and_integrating_into_process() + +``` + + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/vector_databases.md b/docs/components/vector_databases.md new file mode 100644 index 0000000..1d06b03 --- /dev/null +++ b/docs/components/vector_databases.md @@ -0,0 +1,173 @@ +--- +layout: default +title: Vector Databases +parent: Components +nav_order: 11 +description: overview of the major modules and classes of LLMWare +permalink: /components/vector_databases +--- +# Vector Databases +--- + +llmware supports the following vector databases: + + - Milvus and Milvus-Lite - `milvus` + - Postgres (PG Vector) - `postgres` + - Qdrant - `qdrant` + - ChromaDB - `chromadb` + - Redis - `redis` + - Neo4j - `neo4j` + - LanceDB - `lancedb` + - FAISS - `faiss` + - Mongo-Atlas - `mongo-atlas` + - Pinecone - `pinecone` + +In llmware, unstructured content is ingested and organized into a Library, and then embeddings are created against the +Library object, and usually, handled by implicitly through the Library method `.install_new_embedding`. + +All embedding models are implemented through the embeddings.py module, and the `EmbeddingHandler` class, which routes +the embedding process to the vector db specific handler and provides a common set of utility functions. +In most cases, it is not necessarily to explicitly call the vector db class. + +The design is intended to promote code re-use and to make it easy to experiment with different endpoint vector databases +without significant code changes, as well as to leverage the Library as the core organizing construct. + +# Select Vector DB +To select a vector database in llmware is generally done is one of two ways: + +1. Explicit Setting - `LLMWareConfig().set_vector_db("postgres")` + +2. Pass the name of the vector database at the time of installing the embeddings: + + `library.install_new_embedding(embedding_model_name=embedding_model, vector_db='milvus',batch_size=100)` + +# Install Vector DB + +No-install options: chromadb, lancedb, faiss, and milvus-lite + +API-based options: mongo-atlas, pinecone + +Install server options: + +Generally, we have found that Docker (and Docker-Compose) are the easiest and most consistent ways to install vector +db across different platforms. + +1. milvus - we provide a docker-compose script in the main repository root folder path, which installs mongodb as well. + +```bash +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose_mongo_milvus.yaml +docker compose up -d +``` + +2. qdrant + +```bash +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose-qdrant.yaml +docker compose up -d +``` + +3. postgres and pgvector + +```bash +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose-pgvector.yaml +docker compose up -d +``` + +4. redis +```bash +# scripts to deploy other options +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose-redis-stack.yaml +``` + +5. neo4j + +```bash +curl -o docker-compose.yaml https://raw.githubusercontent.com/llmware-ai/llmware/main/docker-compose-neo4j.yaml +docker compose up -d +``` + +# Configure Vector DB + +To configure a vector database in llmware, we provide configuration objects in the `configs.py` module to adjust +authentication, port/host information, and other common configurations. To use the configuration, the pattern is +as follows through simple `get_config` and `set_config` methods: + +```python +from llmware.configs import MilvusConfig +MilvusConfig().set_config("lite", True) + +from llmware.configs import ChromaDBConfig +current_config = ChromaDBConfig().get_config("persistent_path") +ChromaDBConfig().set_config("persistent_path", "/new/local/path") +``` + +Configuration objects are provided for the following vector DB: `MilvusConfig`, `ChromaDBConfig`, `QdrantConfig`, +`Neo4jConfig`, `LanceDBConfig`, `PineConeConfig`, `MongoConfig`, `PostgresConfig`. + +For 'out-of-the-box' testing and development, for most use cases, you will not need to change these configs. + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/components/whisper_cpp.md b/docs/components/whisper_cpp.md new file mode 100644 index 0000000..a1e0bd6 --- /dev/null +++ b/docs/components/whisper_cpp.md @@ -0,0 +1,203 @@ +--- +layout: default +title: Whisper CPP +parent: Components +nav_order: 14 +description: overview of the major modules and classes of LLMWare +permalink: /components/whisper_cpp +--- +# Whisper CPP +--- + +llmware has an integrated WhisperCPP backend which enables fast, easy local voice-to-text processing. + +Whisper is a leading open voice voice-to-text model from OpenAI - https://github.com/openai/whisper + +WhisperCPP is the implementation of Whisper packaged as a GGML deliverable - https://github.com/ggerganov/whisper.cpp + +Starting with llmware 0.2.11, we have integrated WhisperCPPModel as a new model class, +providing options for direct inference, and coming soon, integration into the Parser for easy text chunking and +parsing into a Library with other document types. + +llmware provides prebuilt shared libraries for WhisperCPP on the following platforms: + --Mac M series + --Linux x86 (no CUDA) + --Linux x86 (with CUDA) - really fast + --Windows x86 (only on CPU) currently. + +We have added three Whisper models to the default model catalog: + +1. ggml-base.en.bin - english-only base model +2. ggml-base.bin - multi-lingual base model +3. ggml-small.en-tdrz.bin - this is a 'tiny-diarize' implementation that has been finetuned to identify the +speakers and inserts special [_SOLM_] tags to indicate a conversation turn / change of speaker. + + Main repo: https://github.com/akashmjn/tinydiarize/ + Citation: @software{mahajan2023tinydiarize, + author = {Mahajan, Akash}, month = {08}, + title = {tinydiarize: Minimal extension of Whisper for speaker segmentation with special tokens} + url = {https://github.com/akashmjn/tinydiarize}, + year = {2023} + +To use WAV files, there is one additional Python dependency required: + --pip install librosa + --Note: this has been added to the default requirements.txt and pypy build starting with 0.2.11 + +To use other popular audio/video file formats, such as MP3, MP4, M4A, etc., then the following dependencies are +required: + --pip install pydub + --ffmpeg library - which can be installed as follows: + -- Linux: `sudo apt install ffmpeg' + -- Mac: `brew install ffmpeg` + -- Windows: direct download and install from ffmpeg + + +```python + +""" This example shows how to use llmware provided sample files for testing with WhisperCPP, integrated as of + llmware 0.2.11. + + # examples - "famous_quotes" | "greatest_speeches" | "youtube_demos" | "earnings_calls" + + -- famous_quotes - approximately 20 small .wav files with clips from old movies and speeches + -- greatest_speeches - approximately 60 famous historical speeches in english + -- youtube_videos - wav files of ~3 llmware youtube videos + -- earnings_calls - wav files of ~4 public company earnings calls (gathered from public investor relations) + + These sample files are hosted in a non-restricted AWS S3 bucket, and downloaded via the Setup method + `load_sample_voice_files`. There are two options: + + -- small_only = True: only pulls the 'famous_quotes' samples + -- small_only = False: pulls all of the samples (requires ~1.9 GB in total) + + Please note that all of these samples have been pulled from open public domain sources, including the + Internet Archives, e.g., https://archive.org. These sample files are being provided solely for the purpose of + testing the code scripts below. Please do not use them for any other purpose. + + To run these examples, please make sure to `pip install librosa` + """ + +import os +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs +from llmware.setup import Setup + +# optional / to adjust various parameters of the model +GGUFConfigs().set_config("whisper_cpp_verbose", "OFF") +GGUFConfigs().set_config("whisper_cpp_realtime_display", True) + +# note: english is default output - change to 'es' | 'fr' | 'de' | 'it' ... +GGUFConfigs().set_config("whisper_language", "en") +GGUFConfigs().set_config("whisper_remove_segment_markers", True) + + +def sample_files(example="famous_quotes", small_only=False): + + """ Execute a basic inference on Voice-to-Text model passing a file_path string """ + + voice_samples = Setup().load_voice_sample_files(small_only=small_only) + + examples = ["famous_quotes", "greatest_speeches", "youtube_demos", "earnings_calls"] + + if example not in examples: + print("choose one of the following - ", examples) + return 0 + + fp = os.path.join(voice_samples,example) + + files = os.listdir(fp) + + # these are the two key lines + whisper_base_english = "whisper-cpp-base-english" + + model = ModelCatalog().load_model(whisper_base_english) + + for f in files: + + if f.endswith(".wav"): + + prompt = os.path.join(fp,f) + + print(f"\n\nPROCESSING: prompt = {prompt}") + + response = model.inference(prompt) + + print("\nllm response: ", response["llm_response"]) + print("usage: ", response["usage"]) + + return 0 + + +if __name__ == "__main__": + + # pick among the four examples: famous_quotes | greatest_speeches | youtube_demos | earnings_calls + + sample_files(example="famous_quotes", small_only=False) +``` + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in Oktober 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/contributing/code.md b/docs/contributing/code.md new file mode 100644 index 0000000..d10eaa4 --- /dev/null +++ b/docs/contributing/code.md @@ -0,0 +1,307 @@ +--- +layout: default +title: Code contributions +parent: Contributing +nav_order: 1 +permalink: /contributing/code +--- +# Contributing code +One way to contribute to ``llmware`` is by contributing to the code base. + +We briefly describe some of the important modules of ``llmware`` next, so you can more easily navigate the code base. +You may also take a look at our [fast start series from YouTube](https://www.youtube.com/playlist?list=PL1-dn33KwsmD7SB9iSO6vx4ZLRAWea1DB). + +## Core modules + +### Library + +The *library* module implements the classes **Library** and **LibraryCatalog**. +The **Library** class implements the *library* concept. +A *library* is a collection of documents, where a document can be PDF, an image, or an office document. +It is responsible for parsing, text chunking, and indexing. +In other words, it does the heavy lifting of adding content. +In the following, we shortly describe the functions for adding documents to the library. + +```python +add_file( + self, + file_path): +``` +This method adds one document of any supported type to the library. + +```python +add_files( + self, + input_folder_path=None, + encoding="utf-8", + chunk_size=400, + get_images=True,get_tables=True, + smart_chunking=2, + max_chunk_size=600, + table_grid=True, + get_header_text=True, + table_strategy=1, + strip_header=False, + verbose_level=2, + copy_files_to_library=True): +``` +This method adds the documents of one folder to the library. + +```python +add_website( + self, + url, + get_links=True, + max_links=5): +``` +This method adds a website, and links from the website, to the library. + +```python +add_wiki( + self, + topic_list, + target_results=10): +``` +This method adds a wikipedia article to the library. + +```python +add_dialogs( + self, + input_folder=None): +``` +This method adds an AWS dialog transcript to the library. + +```python +add_image( + self, + input_folder=None): +``` +This method adds images to the libary. + +```python +add_pdf_by_ocr( + self, + input_folder=None): +``` +This method adds scanned PDFs to the library. + +```python +add_pdf( + self, + input_folder=None): +``` +This method adds PDFs to the library. + +```python +add_office( + self, + input_folder=None): +``` +This method adds office documents to the library. + +### Embeddings + +An *embedding* is a vector store and an embedding model. +It is responsible for applying an embedding model to a library, storing the embeddings in a vector store, and providing access to the embeddings with natural language queries. +We briefly describe the common methods offered by all vector stores below. + +```python +def create_new_embedding( + self, + doc_ids=None, + batch_size=500): +``` +This method creates the embeddings and adds them to the vector store. + +```python +def search_index( + self, + query_embedding_vector, + sample_count=10): +``` +This method searches the vector store given the query vector. + +```python +def delete_index(self): +``` +This method deletes the created vector store index. + + +### Prompts + +A *prompt* is an input to model. +The prompt is used by the model to generate the response. +One important use case is that users want to augment a prompt, or a series of prompts, with additional information. +Next, we describe methods for augmenting a prompt with additional information. + +```python +def add_source_new_query( + self, + library, + query=None, + query_type="semantic", + result_count=10): +``` +This method adds the results of the ``query`` to the prompt. + +```python +def add_source_query_results( + self, + query_results): +``` +This method adds previous results from a query as a source to the prompt. + +```python +def add_source_library( + self, + library_name): +``` +This method adds an entire library to the prompt. +We recommend that you only use this when the library is sufficiently small. + +```python +def add_source_wikipedia( + self, + topic, + article_count=3, + query=None): +``` +This method adds wikipedia articles to the prompt based on the provided ``topic``. + +```python +def add_source_yahoo_finance( + self, + ticker=None, + key_list=None): +``` +This method adds a Yahoo finance ticker to the prompt. + +```python +def add_source_knowledge_graph( + self, + library, + query): +``` +This method adds the summary output elements from a knowledge graph based on the provided ``query``. +Please note that this method is experimental, i.e. unstable, and is subject to change dramatically in each new version. + +```python +def add_source_website( + self, + url, + query=None): +``` +This method adds the website pointed to by the ``url`` to the prompt. + +```python +def add_source_document( + self, + input_fp, + input_fn, + query=None): +``` +This method adds a document, or documents, of any supported type to the prompt. +If documents are added, then the ``query`` parameter can be used to filter the documents. + +```python +def add_source_last_interaction_step( + self): +``` +This method adds the last interaction to the prompt. +The use case for this is to enable interactive dialog, i.e. chatting. + +### Model Catalog +A *model catalog* is a collection of models. +In the following, we briefly describe the methods for adding new models to the catalog. + +```python +def register_new_hf_generative_model( + self, + hf_model_name=None, + context_window=2048, + prompt_wrapper="", + display_name=None, + temperature=0.3, + trailing_space="", + link=""): +``` +This method adds a new generative model from hugging face. +Users can therefore add models from hugging face that are unsupported currently. + +```python +def register_sentence_transformer_model( + self, + model_name, + embedding_dims, + context_window, + display_name=None, + link=""): +``` +This method adds a new sentence transformer. + +```python +def register_gguf_model( + self, + model_name, + gguf_model_repo, + gguf_model_file_name, + prompt_wrapper=None, + eos_token_id=0, + display_name=None, + trailing_space="", + temperature=0.3, + context_window=2048, + instruction_following=True): +``` +This method adds a new GGUF model. + +```python +def register_open_chat_model( + cls, + model_name, + api_base=None, + model_type="chat", + display_name=None, + context_window=4096, + instruction_following=True, + prompt_wrapper="", + temperature=0.5): +``` +This method adds any chat model that is available through a web API, e.g. a chat model that is available locally +via localhost. + +```python +def register_ollama_model( + cls, + model_name, + host="localhost", + port=11434, + model_type="chat", + raw=False, + stream=False, + display_name=None, + context_window=4096, + instruction_following=True, + prompt_wrapper="", + temperature=0.5): +``` +This method adds an OLLama model that is available through a web API. +The method is similar to the ``register_open_chat_model`` method above. + +### Categories of code contributions + +#### New or Enhancement to existing Features +You want to submit a code contribution that adds a new feature or enhances an existing one? +Then the best way to start is by opening a discussion in our [GitHub discussions](https://github.com/llmware-ai/llmware/discussions). +Please do this before you work on it, so you do not put effort into it just to realise after submission that +it will not be merged. + +#### Bugs +If you encounter a bug, you can + +- File an issue about the bug. +- Provide a self-contained minimal example that reproduces the bug, which is extremely important. +- Provide possible solutions for the bug. +- Submit a pull a request to fix the bug. + +We encourage you to read [How to create a Minimal, Reproducible Example](https://stackoverflow.com/help/minimal-reproducible-example) from the Stackoverflow helpcenter, and the tag description of [self-container](https://stackoverflow.com/tags/self-contained/info), also from Stackoverflow. diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md new file mode 100644 index 0000000..feec1fd --- /dev/null +++ b/docs/contributing/contributing.md @@ -0,0 +1,94 @@ +--- +layout: default +title: Contributing +nav_order: 7 +has_children: true +description: llmware contributions. +permalink: /contributing +--- + +# Contributing to llmware + +{: .note} +> The contributions to `llmware` are governed by our [Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md). + +{: .warning} +> Have you found a security issue? Then please jump to [Security Vulnerabilities](#security-vulnerabilities). + +On this page, we provide information ``llmware`` contributions. +There are **two ways** on how you can contribute. +The first is by making **code contributions**, and the second by making contributions to the **documentation**. +Please look at our [contribution suggestions](#how-can-you-contribute) if you need inspiration, or take a look at [open issues](#open-issues). + +Contributions to `llmware` are welcome from everyone. +Our goal is to make the process simple, transparent, and straightforward. +We are happy to receive suggestions on how the process can be improved. + +## How can you contribute? + +{: .note} +> If you have never contributed before look for issues with the tag [``good first issue``](https://github.com/llmware-ai/llmware/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). + +The most usual ways to contribute is to add new features, fix bugs, add tests, or add documentation. +You can visit the [issues](https://github.com/llmware-ai/llmware/issues) site of the project and search for tags such as +``bug``, ``enhancement``, ``documentation``, or ``test``. + + +Here is a non exhaustive list of contributions you can make. + +1. Code refactoring +2. Add new text data bases +3. Add new vector data bases +4. Fix bugs +5. Add usage examples (see for example the issues [jupyter notebook - more examples and better support](https://github.com/llmware-ai/llmware/issues/508) and [google colab examples and start up scripts](https://github.com/llmware-ai/llmware/issues/507)) +6. Add experimental features +7. Improve code quality +8. Improve documentation in the docs (what you are reading right now) +9. Improve documentation by adding or updating docstrings in modules, classes, methods, or functions (see for example [Add docstrings](https://github.com/llmware-ai/llmware/issues/219)) +10. Improve test coverage +11. Answer questions in our [Discord channel](https://discord.gg/MhZn5Nc39h), especially in the [technical support forum](https://discord.com/channels/1179245642770559067/1218498778915672194) +12. Post projects in which you use ``llmware`` in our Discord forum [made with llmware](https://discord.com/channels/1179245642770559067/1218567269471486012), ideially with a link to a public GitHub repository + +## Open Issues +If you're interested in existing issues, you can + +- Look for issues, if you are a new to the project, look for issues with the `good first issue` label. +- Provide answers for questions in our [GitHub discussions](https://github.com/llmware-ai/llmware/discussions) +- Provide help for bug or enhancement issues. + - Ask questions, reproduce the issues, or provide solutions. + - Pull a request to fix the issue. + + + +## Security Vulnerabilities +**If you believe you've found a security vulnerability, then please _do not_ submit an issue ticket or pull request or otherwise publicly disclose the issue.** +Please follow the process at [Reporting a Vulnerability](https://github.com/llmware-ai/llmware/blob/main/Security.md) + + + +## GitHub workflow + +We follow the [``fork-and-pull``](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) Git workflow. + +1. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the repository on GitHub. +2. Clone your fork to your local machine with `git clone git@github.com:/llmware.git`. +3. Create a branch with `git checkout -b my-topic-branch`. +4. Run the test suite by navigating to the tests/ folder and running ```./run-tests.py -s``` to ensure there are no failures +5. [Commit](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/committing-changes-to-a-pull-request-branch-created-from-a-fork) changes to your own branch, then push to GitHub with `git push origin my-topic-branch`. +6. Submit a [pull request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) so that we can review your changes. + +Remember to [synchronize your forked repository](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo#keep-your-fork-synced) _before_ submitting proposed changes upstream. If you have an existing local repository, please update it before you start, to minimize the chance of merge conflicts. + +```shell +git remote add upstream git@github.com:llmware-ai/llmware.git +git fetch upstream +git checkout upstream/main -b my-topic-branch +``` + +## Community +Questions and discussions are welcome in any shape or form. +Please fell free to join our community on our discord channel, on which we are active daily. +You are also welcome if you just want to post an idea! + +- [Discord Channel](https://discord.gg/MhZn5Nc39h) +- [GitHub discussions](https://github.com/llmware-ai/llmware/discussions) diff --git a/docs/contributing/documentation.md b/docs/contributing/documentation.md new file mode 100644 index 0000000..f41c752 --- /dev/null +++ b/docs/contributing/documentation.md @@ -0,0 +1,66 @@ +--- +layout: default +title: Documentation contributions +parent: Contributing +nav_order: 2 +permalink: contributing/documentation +--- +# Contributing documentation +One way to contribute to ``llmware`` is by contributing documentation. + +There are **two ways** to contribute to the ``llmware`` documentation. +The first is via **docstrings in the code**, and the second is **the docs**, which is what you are *currently reading*. +In both areas, you can contribute in a lot of ways. +Here is a non exhaustive list of these ways for the docstrings which also apply to the docs. + +1. Add documentation (e.g., adding a docstring to a function) +2. Update documentation (e.g., update a docstring that is not in sync with the code) +3. Simplify documentation (e.g., formulate a docstring more clearly) +4. Enhance documentation (e.g., add more examples to a docstring or fix typos) + +## Docstrings +**Docstrings** document the code within the code, which allows programmers to easily have a look while they are programming. +For an exmaple, have a look at [this docstring](https://github.com/llmware-ai/llmware/blob/c9e12a7a150162986622738e127c37ac70f31cd6/llmware/agents.py#L27-L66) which documents the ``LLMfx`` class. + +We follow the docstring style of **numpy**, for which you can find an example [here](https://github.com/numpy/numpydoc/blob/main/doc/example.py) and [here](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_numpy.html). +Please be sure to follow the conventions and go over your pull request before you submit it. + + +## Docs + +{: .note} +> All commands are executed from the `docs` sub-directory. + +Contributing to this documentation is extremely important as many users will refer to it. + +If you plan to contribute to the docs, we recommend that you locally install `jekyll` so you can test your changes locally. +We also recommend, that you install `jekyll` into a a ruby enviroment so it does not interfere with any other installations you might have. + +We recommend that you install `rbenv` and `rvm` to manage your ruby installation. +`rbenv` is a tool that mangages different ruby versions, similar to what `conda` does for `python`. +Please [install rbenv](https://github.com/rbenv/rbenv?tab=readme-ov-file#installation) following their instructions, and the same for [install rvm](https://github.com/rvm/rvm?tab=readme-ov-file#installing-rvm). +We recommend that you install a ruby version `>=3.0`. +After having installed an isolated ruby version, you have to install the dependencies to build the docs locally. +The `docs` directory has a `Gemfile` which specifies the dependencies. +You can hence simply navigate to it and use the `bundle install` command. + +```bash +bundle install +``` + +You should now be able to build and serve the documentation locally. +To do this, simply to the following. +```bash +bundle exec jekyll server --livereload --verbose +``` +In the browser of your choice, you can then go to `http://127.0.0.1:4000/` and you will be served the documentation, which is re-build and re-loaded after any change to the `docs`. +``jekyll`` will create a ``_site`` directory where it saves the created files, please **never commit any files from the \_site directory**! + +## Open Issues +If you're interested in existing issues, you can + +- Look for issues with the `good first issue` and `documentation` label as a good place to get started. +- Provide answers for questions in our [GitHub discussions](https://github.com/llmware-ai/llmware/discussions) +- Provide help for bug or enhancement issues. + - Ask questions, reproduce the issues, or provide solutions. + - Pull a request to fix the issue. diff --git a/docs/examples/agents.md b/docs/examples/agents.md new file mode 100644 index 0000000..cd649a4 --- /dev/null +++ b/docs/examples/agents.md @@ -0,0 +1,98 @@ +--- +layout: default +title: Agents +parent: Examples +nav_order: 2 +description: overview of the major modules and classes of LLMWare +permalink: /examples/agents +--- +# Agents + + + 🚀 Start Building Multi-Model Agents Locally on a Laptop 🚀 +=============== + +**What is a SLIM?** + +**SLIMs** are **S**tructured **L**anguage **I**nstruction **M**odels, which are small, specialized 1-3B parameter LLMs, +finetuned to generate structured outputs (Python dictionaries and lists, JSON and SQL) that can be handled programmatically, and +stacked together in multi-step, multi-model Agent workflows - all running on a local CPU. + +**New SLIMS Just released** - check out slim-extract, slim-summarize, slim-xsum, slim-sa-ner, slim-boolean and slim-tags-3b + +**Check out the new examples below marked with ⭐** +🔥🔥🔥 Web Services & Function Calls ([code](web_services_slim_fx.py)) 🔥🔥🔥 + +**Check out the Intro videos** +[SLIM Intro Video](https://www.youtube.com/watch?v=cQfdaTcmBpY) + +There are 16 SLIM models, each delivered in two packages - a Pytorch/Huggingface FP16 model, and a +quantized "tool" designed for fast inference on a CPU, using LLMWare's embedded GGUF inference engine. In most cases, +we would recommend that you start with the "tools" version of the models. + +**Getting Started** + +We have several ready-to-run examples in this repository: + +| Example | Detail | +|-----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------| +| 1. Getting Started with SLIM Models ([code](slims-getting-started.py) / [video](https://www.youtube.com/watch?v=aWZFrTDmMPc&t=196s)) | Install the models and run hello world tests to see the models in action. | +| 2. Getting Started with Function-Calling Agent ([code](agent-llmfx-getting-started.py) / [video](https://www.youtube.com/watch?v=cQfdaTcmBpY)) | Generate a Structured Report with LLMfx | +| 3. Multi-step Complex Analysis with Agent ([code](agent-multistep-analysis.py) / [video](https://www.youtube.com/watch?v=y4WvwHqRR60)) | Delivering Complex Research Analysis with SLIM Agents | | +| 4. Document Clustering ([code](document-clustering.py)) | Multi-faceted automated document analysis with Topics, Tags and NER | +| 5. Two-Step NER Retrieval ([code](ner-retrieval.py)) | Using NER to extract name, and then using as basis for retrieval. | | +| 6. Using Sentiment Analysis ([code](sentiment-analysis.py)) | Using sentiment analysis on earnings transcripts and a 'if...then' condition | +| 7. Text2SQL - Intro ([code](text2sql-getting-started-1.py)) | Getting Started with SLIM-SQL-TOOL and Basic Text2SQL Inference | | +| 8. Text2SQL - E2E ([code](text2sql-end-to-end-2.py)) | End-to-End Natural Langugage Query to SQL DB Query | | +| 9. Text2SQL - MultiStep ([code](text2sql-multistep-example-3.py)) | Extract a customer name using NER and use in a Text2SQL query | +| 10. ⭐ Web Services & Function Calls ([code](web_services_slim_fx.py)) | Generate 30 key financial analysis with SLIM function calls and web services | +| 11. ⭐ Yes-No Questions with Explanations ([code](using_slim_boolean_model.py)) | Analyze earnings releases with SLIM Boolean | +| 12. ⭐ Extracting Revenue Growth ([code](using_slim_extract_model.py)) | Extract revenue growth from earnings releases with SLIM Extract | +| 13. ⭐ Summary as a Function Call ([code](using_slim_summary.py)) | Simple Summarization as a Function Call with List Length Parameters | +| 14. ⭐ Handling Not Found Extracts ([code](not_found_extract_with_lookup.py)) | Multi-step Lookup strategy and handling not-found answers | +| 15. ⭐ Extract + Lookup ([code](custom_extract_and_lookup.py)) | Extract Named Entity information and use for lookups with SLIM Extract | +| 16. ⭐ Headline/Title as XSUM Function Call ([code](using_slim_xsum.py)) | eXtreme Summarization (XSUM) with SLIM XSUM | + +For information on all of the SLIM models, check out [LLMWare SLIM Model Collection](https://www.huggingface.co/llmware/). + +**Models List** +If you would like more information about any of the SLIM models, please check out their model card: + +- extract - extract custom keys - [slim-extract](https://www.huggingface.co/llmware/slim-extract) & [slim-extract-tool](https://www.huggingface.co/llmware/slim-extract-tool) +- summary - summarize function call - [slim-summary](https://www.huggingface.co/llmware/slim-summary) & [slim-summary-tool](https://www.huggingface.co/llmware/slim-summary-tool) +- xsum - title/headline function call - [slim-xsum](https://www.huggingface.co/llmware/slim-xsum) & [slim-xsum-tool](https://www.huggingface.co/llmware/slim-xsum-tool) +- ner - extract named entities - [slim-ner](https://www.huggingface.co/llmware/slim-ner) & [slim-ner-tool](https://www.huggingface.co/llmware/slim-ner-tool) +- sentiment - evaluate sentiment - [slim-sentiment](https://www.huggingface.co/slim-sentiment) & [slim-sentiment-tool](https://www.huggingface.co/llmware/slim-sentiment-tool) +- topics - generate topic - [slim-topics](https://www.huggingface.co/slim-topics) & [slim-topics-tool](https://www.huggingface.co/llmware/slim-topics-tool) +- sa-ner - combo model (sentiment + named entities) - [slim-sa-ner](https://www.huggingface.co/slim-sa-ner) & [slim-sa-ner-tool](https://www.huggingface.co/llmware/slim-sa-ner-tool) +- boolean - provides a yes/no output with explanation - [slim-boolean](https://www.huggingface.co/slim-boolean) & [slim-boolean-tool](https://www.huggingface.com/llmware/slim-boolean-tool) +- ratings - apply 1 (low) - 5 (high) rating - [slim-ratings](https://www.huggingface.co/slim-ratings) & [slim-ratings-tool](https://www.huggingface.co/llmware/slim-ratings-tool) +- emotions - assess emotions - [slim-emotions](https://www.huggingface.co/slim-emotions) & [slim-emotions-tool](https://www.huggingface.co/llmware/slim-emotions-tool) +- tags - auto-generate list of tags - [slim-tags](https://www.huggingface.co/slim-tags) & [slim-tags-tool](https://www.huggingface.co/llmware/slim-tags-tool) +- tags-3b - enhanced auto-generation tagging model - [slim-tags-3b](https://www.huggingface.com/slim-tags-3b) & [slim-tags-3b-tool](https://www.huggingface.co/llmware/slim-tags-3b-tool) +- intent - identify intent - [slim-intent](https://www.huggingface.co/slim-intent) & [slim-intent-tool](https://www.huggingface.co/llmware/slim-intent-tool) +- category - high-level category - [slim-category](https://www.huggingface.co/slim-category) & [slim-category-tool](https://wwww.huggingface.co/llmware/slim-category-tool) +- nli - assess if evidence supports conclusion - [slim-nli](https://www.huggingface.co/slim-nli) & [slim-nli-tool](https://www.huggingface.co/llmware/slim-nli-tool) +- sql - convert text into sql - [slim-sql](https://www.huggingface.co/slim-sql) & [slim-sql-tool](https://www.huggingface.co/llmware/slim-sql-tool) + +You may also want to check out these quantized 'answer' tools, which work well in conjunction with SLIMs for question-answer and summarization: +- bling-stablelm-3b-tool - 3b quantized RAG model - [bling-stablelm-3b-gguf](https://www.huggingface.co/llmware/bling-stablelm-3b-gguf) +- bling-answer-tool - 1b quantized RAG model - [bling-answer-tool](https://www.huggingface.co/llmware/bling-answer-tool) +- dragon-yi-answer-tool - 6b quantized RAG model - [dragon-yi-answer-tool](https://www.huggingface.co/llmware/dragon-yi-answer-tool) +- dragon-mistral-answer-tool - 7b quantized RAG model - [dragon-mistral-answer-tool](https://www.huggingface.co/llmware/dragon-mistral-answer-tool) +- dragon-llama-answer-tool - 7b quantized RAG model - [dragon-llama-answer-tool](https://www.huggingface.co/llmware/dragon-llama-answer-tool) + + +**Set up** +No special setup for SLIMs is required other than to install llmware >=0.2.6, e.g., `pip3 install llmware`. + +**Platforms:** +- Mac M1, Mac x86, Windows, Linux (Ubuntu 22 preferred, supported on Ubuntu 20 +) +- RAM: 16 GB minimum +- Python 3.9, 3.10, 3.11 (note: not supported on 3.12 yet) +- llmware >= 0.2.6 version + + +### **Let's get started! 🚀** + + diff --git a/docs/examples/datasets.md b/docs/examples/datasets.md new file mode 100644 index 0000000..ca0e15d --- /dev/null +++ b/docs/examples/datasets.md @@ -0,0 +1,134 @@ +--- +layout: default +title: Datasets +parent: Examples +nav_order: 10 +description: overview of the major modules and classes of LLMWare +permalink: /examples/datasets +--- +# Datasets - Introduction by Examples + +llmware provides powerful capabilities to transform raw unstructured information into various model-ready datasets. + +```python + +import os +import json + +from llmware.library import Library +from llmware.setup import Setup +from llmware.dataset_tools import Datasets +from llmware.retrieval import Query + +def build_and_use_dataset(library_name): + + # Setup a library and build a knowledge graph. Datasets will use the data in the knowledge graph + print (f"\n > Creating library {library_name}...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + library.generate_knowledge_graph() + + # Create a Datasets object from library + datasets = Datasets(library) + + # Build a basic dataset useful for industry domain adaptation for fine-tuning embedding models + print (f"\n > Building basic text dataset...") + + basic_embedding_dataset = datasets.build_text_ds(min_tokens=500, max_tokens=1000) + dataset_location = os.path.join(library.dataset_path, basic_embedding_dataset["ds_id"]) + + print (f"\n > Dataset:") + print (f"(Files referenced below are found in {dataset_location})") + + print (f"\n{json.dumps(basic_embedding_dataset, indent=2)}") + sample = datasets.get_dataset_sample(datasets.current_ds_name) + + print (f"\nRandom sample from the dataset:\n{json.dumps(sample, indent=2)}") + + # Other Dataset Generation and Usage Examples: + + # Build a simple self-supervised generative dataset- extracts text and splits into 'text' & 'completion' + # Several generative "prompt_wrappers" are available - chat_gpt | alpaca | + basic_generative_completion_dataset = datasets.build_gen_ds_targeted_text_completion(prompt_wrapper="alpaca") + + # Build a generative self-supervised training sets created by pairing 'header_text' with 'text' + xsum_generative_completion_dataset = datasets.build_gen_ds_headline_text_xsum(prompt_wrapper="human_bot") + topic_prompter_dataset = datasets.build_gen_ds_headline_topic_prompter(prompt_wrapper="chat_gpt") + + # Filter a library by a key term as part of building the dataset + filtered_dataset = datasets.build_text_ds(query="agreement", filter_dict={"master_index":1}) + + # Pass a set of query results to create a dataset from those results only + query_results = Query(library=library).query("africa") + query_filtered_dataset = datasets.build_text_ds(min_tokens=250,max_tokens=600, qr=query_results) + + return 0 +``` + +For more examples, see the [datasets example]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Datasets/) in the main repo. + + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/examples/embedding.md b/docs/examples/embedding.md new file mode 100644 index 0000000..6ca6267 --- /dev/null +++ b/docs/examples/embedding.md @@ -0,0 +1,222 @@ +--- +layout: default +title: Embedding +parent: Examples +nav_order: 5 +description: overview of the major modules and classes of LLMWare +permalink: /examples/embedding +--- +# Embedding - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + +```python + +""" This example is a fast start with Milvus Lite, which is a 'no-install' file-based version of Milvus, intended +for rapid prototyping. A couple of key points to note: + + -- Platform - per Milvus docs, Milvus Lite is designed for Mac and Linux (not on Windows currently) + -- PyMilvus - need to `pip install pymilvus>=2.4.2` + -- within LLMWare: set MilvusConfig().set_config("lite", True) +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.status import Status +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig, MilvusConfig + +from importlib import util + +if not util.find_spec("pymilvus"): + print("\nto run this example with pymilvus, you need to install pymilvus: pip3 install pymilvus>=2.4.2") + + +def setup_library(library_name): + + """ Note: this setup_library method is provided to enable a self-contained example to create a test library """ + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # check the embedding status 'before' installing the embedding + embedding_record = library.get_embedding_status() + print("embedding record - before embedding ", embedding_record) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in database + + print("update: Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements"), + chunk_size=400, max_chunk_size=600, smart_chunking=1) + + return library + + +def install_vector_embeddings(library, embedding_model_name): + + """ This method is the core example of installing an embedding on a library. + -- two inputs - (1) a pre-created library object and (2) the name of an embedding model """ + + library_name = library.library_name + vector_db = LLMWareConfig().get_vector_db() + + print(f"\nupdate: Starting the Embedding: " + f"library - {library_name} - " + f"vector_db - {vector_db} - " + f"model - {embedding_model_name}") + + # *** this is the one key line of code to create the embedding *** + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db,batch_size=100) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + # Start using the new vector embeddings with Query + sample_query = "incentive compensation" + print("\n\nupdate: Run a sample semantic/vector query: {}".format(sample_query)) + + # queries are constructed by creating a Query object, and passing a library as input + query_results = Query(library).semantic_query(sample_query, result_count=20) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # to see all of the dictionary keys returned, uncomment the line below + # print("update: query_results - all - ", i, entries) + + # for display purposes only, we will only show the first 125 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + # lets take a look at the library embedding status again at the end to confirm embeddings were created + embedding_record = library.get_embedding_status() + + print("\nupdate: embedding record - ", embedding_record) + + return 0 + + +if __name__ == "__main__": + + # Fast Start configuration - will use no-install embedded sqlite + # -- if you have installed Mongo or Postgres, then change the .set_active_db accordingly + + LLMWareConfig().set_active_db("sqlite") + + # set the "lite" flag in MilvusConfig to True -> to use server version, set to False (which is default) + MilvusConfig().set_config("lite", True) + LLMWareConfig().set_vector_db("milvus") + + # Step 1 - create library + library = setup_library("ex2_milvus_lite") + + # Step 2 - Select any embedding model in the LLMWare catalog + + # to see a list of the embedding models supported, uncomment the line below and print the list + embedding_models = ModelCatalog().list_embedding_models() + + # for i, models in enumerate(embedding_models): + # print("embedding models: ", i, models) + + # for this first embedding, we will use a very popular and fast sentence transformer + embedding_model = "mini-lm-sbert" + + # note: if you want to swap out "mini-lm-sbert" for Open AI 'text-embedding-ada-002', uncomment these lines: + # embedding_model = "text-embedding-ada-002" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # run the core script + install_vector_embeddings(library, embedding_model) +``` + + +For more examples, see the [embedding examples]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Embedding/) in the main repo. + + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/examples/examples.md b/docs/examples/examples.md new file mode 100644 index 0000000..338d701 --- /dev/null +++ b/docs/examples/examples.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Examples +nav_order: 5 +has_children: true +description: examples, recipes and use cases +permalink: /examples +--- + +llmware offers a wide range of examples to cover the lifecycle of building RAG and Agent based applications using +small language models: + + - [Parsing examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing) - ~14 stand-alone parsing examples for all common document types, including options for parsing in memory, outputting to JSON, parsing custom configured CSV and JSON files, running OCR on embedded images found in documents, table extraction, image extraction, text chunking, zip files, and web sources. + - [Embedding examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Embedding) - ~15 stand-alone embedding examples to show how to use ~10 different vector databases and wide range of leading open source embedding models (including sentence transformers). + - [Retrieval examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Retrieval) - ~10 stand-alone examples illustrating different query and retrieval techniques - semantic queries, text queries, document filters, page filters, 'hybrid' queries, author search, using query state, and generating bibliographies. + - [Dataset examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Datasets) - ~5 stand-alone examples to show 'next steps' of how to leverage a Library to re-package content into various datasets and automated NLP analytics. + - [Fast start example #1-Parsing](https://github.com/llmware-ai/llmware/blob/main/fast_start/rag/example-1-create_first_library.py) - shows the basics of parsing. + - [Fast start example #2-Embedding](https://github.com/llmware-ai/llmware/blob/main/fast_start/rag/example-2-build_embeddings.py) - shows the basics of building embeddings. + - [CustomTable examples](https://github.com/llmware-ai/llmware/tree/main/examples/Structured_Tables) - ~5 examples to start building structured tables that can be used in conjunction with LLM-based workflows. + + - [Models examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Models) - ~20 examples showing a wide range of different model inferences and use cases, including the ability to integrate Ollama models, OpenChat (e.g., LMStudio) models, using LLama-3 and Phi-3, bringing your own models into the ModelCatalog, and configuring sampling settings. + - [Prompts examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Prompts) - ~5 examples that illustrate how to use Prompt as an integrated workflow for integrating knowledge sources, managing prompt history, and applying fact-checking. + - [SLIM-Agents examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/SLIM-Agents) - ~20 examples showing how to build multi-model, multi-step Agent processes using locally-running SLIM function calling models. + - [Fast start example #3-Prompts and Models](https://github.com/llmware-ai/llmware/blob/main/fast_start/rag/example-3-prompts_and_models.py) - getting started with model inference. + diff --git a/docs/examples/getting_started.md b/docs/examples/getting_started.md new file mode 100644 index 0000000..af8dbfb --- /dev/null +++ b/docs/examples/getting_started.md @@ -0,0 +1,124 @@ +--- +layout: default +title: Introduction by Examples +parent: Examples +nav_order: 9 +permalink: /examples/getting_started +--- +# Introduction by Examples +We introduce ``llmware`` through self-contained examples. + + +# Your first library and query + +{: .note } +> The code here is a modified version from [example-1-create_first_library.py](https://github.com/llmware-ai/llmware/blob/main/fast_start/example-1-create_first_library.py). +> The adjustments are made to ease understanding for this post. + +In this introduction, we will walk through the steps of creating a **library**. +To create a ``library`` in ``llmware`` we have to instantiate a ``library`` object and call +the ``add_files`` method, which will parse the files, chunk up the text and also index it. +We will also download the samples files we provide, which can be used for any experimentation you +might want to do. + + +**Configuring llmware** + +Before we get started, we can influence the configuration of ``llmware``. +For example, we can decide on which **text collection** data base to use, and on the logging level. +By default, ``llmware`` uses MongoDB as the text collection data base and has a ``debug_mode`` level +of ``0``. +This means that by default, ``llmware`` will show the status manager and print errors. +The status manager is useful for large parsing jobs. +In this ``library`` introduction, we will change the text collection data base as well as the ``debug_mode``. +As the text collection data base, we will choose ``sqlite``. +And we will change the ``debug_mode`` to ``2``, which will show the file name that is being parsed, i.e. a file-by-file progress. +```python +from llmware.configs import LLMWareConfig + +LLMWareConfig().set_active_db("sqlite") +LLMWareConfig().set_config("debug_mode", 2) +``` + +**Downloading sample files** + +We start by downloading the sample files we need. +``llmware`` provides a set of sample files which we use throughout our examples. +The following code snippet downloads these sample files, and in doing so creates the directories +*Agreements*, *Invoices*, *UN-Resolutions-500*, *SmallLibrary*, *FinDocs*, and *AgreementsLarge*. +If you want to get the newest version of the sample files, you can set ``over_write=True``. +However, we encourage you to try it out with your own files once you are comfortable enough with ``llmware``. +```python +from llmware.setup import Setup + +sample_files_path = Setup().load_sample_files(over_write=False) +``` +``sample_files_path`` is the path where the files are stores. +Assume that your use name is ``foo``, then on Linux the path would be ``'/home/foo/llmware_data/sample_files'.`` + + +**Creating a library** + +Now that we have data, we can start to create our library. +In ``llmware``, a **library** is a collection of unstructured data. +Currently, ``llmware`` supports *text* and *images*. +The following code creates an empty ``library`` with the name ``my_llmware_library``. +```python +from llmware.library import Library + +library = Library().create_new_library('my_llmware_library') +``` + +**Adding files to a library** + +Now that we have created a ``library``, we are ready to *add files* to it. +Currently, the ``add_files`` method supports pdf, pptx, docx, xlsx, csv, md, txt, json, wav, and zip, jpg, and png. +The method will automatically choose the correct parser, based on the file extension. +```python +library.add_files('/home/foo/llmware_data/sample_files/Agreements') +``` + +**The library card** + +A ``library`` keeps inventory of its files, similar to a good librarian. +We do this with a *library card*. +At the moment of this writing, a library card has the keys _id, library_name, embedding, knowledge_graph, unique_doc_id, documents, blocks, images, pages, tables, and account_name. +```python +updated_library_card = library.get_library_card() +doc_count = updated_library_card["documents"] +block_count = updated_library_card["blocks"] +library_card.keys() +``` + +You can also get where the library is stored via the ``library_main_path`` attribute. +Again, assuming your user name is *foo* and you are on a Linux system, then the ``library_path`` is ``'/home/foo/llmware_data/accounts/llmware/my_lib'``. +```python +library.library_main_path +``` + +**Querying a library** + +Finally, we are ready to execute a query against our library. +Remember that the text is indexed automatically when we add it to the library. +The result of a ``Query`` is a list of dictionaries, where one dictionary is one result. +A result dictionary has a wide range of useful keys. +A few important keys in the dictionary are *text*, *file_source*, *page_num*, *doc_ID*, *block_ID*, and +*matches*. +In the following, we query the library for the base salary, return the first ten results, and +iterate over the results. +```python +query_results = Query(library).text_query('base salary', result_count=10) + +for query_result in query_results: + text = query_result["text"] + file_source = query_result["file_source"] + page_number = query_result["page_num"] + doc_id = query_result["doc_ID"] + block_id = query_result["block_ID"] + matches = query_result["matches"] +``` + +You can take a look at all the keys that are returned by calling ``keys()``. +```python +query_results[0].keys() +``` diff --git a/docs/examples/models.md b/docs/examples/models.md new file mode 100644 index 0000000..d009991 --- /dev/null +++ b/docs/examples/models.md @@ -0,0 +1,498 @@ +--- +layout: default +title: Models +parent: Examples +nav_order: 3 +description: overview of the major modules and classes of LLMWare +permalink: /examples/models +--- +# Models + +We introduce ``llmware`` through self-contained examples. + +```python + + +""" This example demonstrates prompting local BLING models with provided context - easy to select among different +BLING models between 1B - 4B, including both Pytorch versions and GGUF quantized versions, and to swap out the +hello_world questions with your own test set. + + NOTE: if you are running on a CPU with limited memory (e.g., <16 GB of RAM), we would recommend sticking to +the 1B parameter models, or using the quantized GGUF versions. You may get out-of-memory errors and/or very +slow performance with ~3B parameter Pytorch models. Even with 16 GB+ of RAM, the 3B Pytorch models should run but +will be slow (without GPU acceleration). """ + + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +def bling_meets_llmware_hello_world (model_name): + + """ Simple inference loop that loads a model and runs through a series of test questions. """ + + t0 = time.time() + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"] + , prompt_name="default_with_context",temperature=0.30) + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # list of 'rag-instruct' laptop-ready bling models on HuggingFace + + model_list = ["llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3", + + # use GGUF models too + "bling-phi-3-gguf", # quantized bling-phi-3 + "bling-answer-tool", # quantized bling-tiny-llama + "bling-stablelm-3b-tool" # quantized bling-stablelm-3b + ] + + # try the newest bling model - 'tiny-llama' + bling_meets_llmware_hello_world(model_list[1]) +``` + +For more examples, see the [models examples]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Models/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/examples/notebooks.md b/docs/examples/notebooks.md new file mode 100644 index 0000000..4165ddf --- /dev/null +++ b/docs/examples/notebooks.md @@ -0,0 +1,52 @@ +--- +layout: default +title: Notebooks +parent: Examples +nav_order: 11 +description: overview of the major modules and classes of LLMWare +permalink: /examples/notebooks +--- +# Notebooks - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + +# Understanding Google Colab and Jupyter Notebooks + +Welcome to our project documentation! A common point of confusion among developers new to data science and machine learning workflows is the relationship and differences between Google Colab and Jupyter Notebooks. This README aims to clarify these parts to ensure everyone is on the same page. + +## What are Jupyter Notebooks? + +Jupyter Notebooks is an open-source web application that lets you create and share documents that have live code, equations, visualizations, and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, machine learning, and much more. + +## What is Google Colab? + +Google Colab (or Colaboratory) is a free Jupyter notebook environment that requires no setup and runs in the cloud. It offers a similar interface to Jupyter Notebooks and lets users write and execute Python in a web browser. Google Colab also provides free access to computing resources, including GPUs and TPUs, making it highly popular for machine learning and data analysis projects. + +## Key Similarities + +- **Interface:** Both platforms use the Jupyter Notebook interface, which supports mixing executable code, equations, visualizations, and narrative text in a single document. +- **Language Support:** Primarily, both are used for executing Python code. However, Jupyter Notebooks support other languages such as R and Julia. +- **Use Cases:** They are widely used for data analysis, machine learning, and education, allowing for easy sharing of results and methodologies. + +## Increase Google Colab Computational Power with T4 GPU + +Our models are designed to run on at least 16GB of RAM. By default Google Colab provides ~13GB of RAM, which significantly slows computational speed. To ensure the best performance when using our models, we highly recommend enabling the T4 GPU in Colab. This will provide the notebook with additional resources, including 16GB of RAM, allowing our models to run smoothly and efficiently. + +Steps to enabling T4 GPU in Colab: +1. In your Colab notebook, click on the "Runtime" tab +2. Select "Change runtime type" +3. Under "Hardware Accelerator", select T4 GPU + +NOTE: There is a weekly usage limit on using T4 for free. + +## Key Differences + +- **Execution Environment:** Jupyter Notebooks can be run locally on your machine or on a server, but Google Colab is hosted in the cloud. +- **Access to Resources:** Google Colab provides free access to hardware accelerators (GPUs and TPUs) which is not inherently available in Jupyter Notebooks unless specifically set up by the user on their servers. +- **Collaboration:** Google Colab offers easier collaboration features, similar to Google Docs, letting multiple users work on the same notebook simultaneously. + +## Conclusion + +While Google Colab and Jupyter Notebooks might seem different they are built on the same idea and offer similar functionalities with a few distinctions, mainly in execution environment and access to computing resources. Understanding these platforms' capabilities can significantly enhance your data science and machine learning projects. + +We hope this guide has helped clarify the similarities and differences between Google Colab and Jupyter Notebooks. Happy coding! + diff --git a/docs/examples/parsing.md b/docs/examples/parsing.md new file mode 100644 index 0000000..f4ae655 --- /dev/null +++ b/docs/examples/parsing.md @@ -0,0 +1,69 @@ +--- +layout: default +title: Parsing +parent: Examples +nav_order: 4 +description: overview of the major modules and classes of LLMWare +permalink: /examples/parsing +--- +# Parsing - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + + +🚀 Parsing Examples 🚀 +=============== + +**Parsing is the Humble Hero of Good RAG Pipelines** + +LLMWare supports parsing of a wide range of unstructured content types, and views parsing, text chunking and indexing as the first step in the pipeline, and like any pipeline, care and attention to getting "great input" is usually the key to "great output." + +In this repository, we show several key features of parsing with llmware: + + +**Parsing PDFs like a Pro** + +- Configuring text chunking and extraction parameters - [**PDF Configuration**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/pdf_parser_new_configs.py) + +- PDF Table extraction - [**PDF Table**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/pdf_table_extraction.py) + +- Fallback to OCR - [**PDF-by-OCR**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_pdf_by_ocr.py) + + +**Parsing Office Documents (Powerpoints, Word, Excel)** + +- Configuring text chunking and extraction parameters - [**Office Configuration**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/office_parser_new_configs.py) + +- Handling ZIPs and mixed file types - [**Microsoft IR Documents**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parsing_microsoft_ir_docs.py) + +- Running OCR on Images Extracted - [**OCR Embedded Doc Images**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/ocr_embedded_doc_images.py) + + +**Parsing without a Database** + +- Parse in Memory - [**Parse in Memory**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_in_memory.py) + +- Parse directly into a Prompt - [**Parse in Prompt**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_into_prompt.py) + +- Parse to JSON file - [**Parse to JSON**](https://www.github.com/llmware-ai/llmware/tree/main/examples/main/examples/Parsing/parse_to_json.py) + + +**Other Content Types** + +- Custom CSV - [**Custom CSV files**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_csv_custom.py) + +- Custom JSON - [**Custom JSON files**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_jsonl_custom.py) + +- Images - [**OCR on Images**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_images.py) + +- Web/HTML - [**Website Extraction**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_web_sources_in_memory.py) + +- Voice (WAV) - in Use_Cases - [**Parsing Great Speeches**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/parsing_great_speeches.py) + +For more examples, see the [parsing examples]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + +### **Let's get started! 🚀** + + diff --git a/docs/examples/prompts.md b/docs/examples/prompts.md new file mode 100644 index 0000000..8a2a206 --- /dev/null +++ b/docs/examples/prompts.md @@ -0,0 +1,263 @@ +--- +layout: default +title: Prompts +parent: Examples +nav_order: 6 +description: overview of the major modules and classes of LLMWare +permalink: /examples/prompts +--- +# Prompts - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + +# Basic RAG Scenario - Invoice Processing + +```python + +""" This example shows an end-to-end scenario for invoice processing that can be run locally and without a +database. The example shows how to combine the use of parsing combined with prompts_with_sources to rapidly +iterate through a batch of invoices and ask a set of questions, and then save the full output to both +(1) .jsonl for integration into an upstream application/database and (2) to a CSV for human review in excel. + + note: the sample code pulls from a public repo to load the sample invoice documents the first time - + please feel free to substitute with your own invoice documents (PDF/DOCX/PPTX/XLSX/CSV/TXT) if you prefer. + + this example does not require a database or embedding + + this example can be run locally on a laptop by setting 'run_on_cpu=True' + if 'run_on_cpu==False", then please see the example 'launch_llmware_inference_server.py' + to configure and set up a 'pop-up' GPU inference server in just a few minutes +""" + +import os +import re + +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.configs import LLMWareConfig +from llmware.setup import Setup +from llmware.models import ModelCatalog + + +def invoice_processing(run_on_cpu=True): + + # Step 1 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + invoices_path = os.path.join(sample_files_path, "Invoices") + + # Step 2 - simple sample query list - each question will be asked to each invoice + query_list = ["What is the total amount of the invoice?", + "What is the invoice number?", + "What are the names of the two parties?"] + + # Step 3 - Load Model + + if run_on_cpu: + + # load local bling model that can run on cpu/laptop + + # note: bling-1b-0.1 is the *fastest* & *smallest*, but will make more errors than larger BLING models + # model_name = "llmware/bling-1b-0.1" + + # try the new bling-phi-3 quantized with gguf - most accurate + model_name = 'bling-phi-3-gguf' + else: + + # use GPU-based inference server to process + # *** see the launch_llmware_inference_server.py example script to setup *** + + server_uri_string = "http://11.123.456.789:8088" # insert your server_uri_string + server_secret_key = "demo-test" + ModelCatalog().setup_custom_llmware_inference_server(server_uri_string, secret_key=server_secret_key) + model_name = "llmware-inference-server" + + # attach inference server to prompt object + prompter = Prompt().load_model(model_name) + + # Step 4 - main loop thru folder of invoices + + for i, invoice in enumerate(os.listdir(invoices_path)): + + # just in case (legacy on mac os file system - not needed on linux or windows) + if invoice != ".DS_Store": + + print("\nAnalyzing invoice: ", str(i + 1), invoice) + + for question in query_list: + + # Step 4A - parses the invoices in memory and attaches as a source to the Prompt + source = prompter.add_source_document(invoices_path,invoice) + + # Step 4B - executes the prompt on the LLM (with the loaded source) + output = prompter.prompt_with_source(question,prompt_name="default_with_context") + + for i, response in enumerate(output): + print("LLM Response - ", question, " - ", re.sub("[\n]"," ", response["llm_response"])) + + prompter.clear_source_materials() + + # Save jsonl report with full transaction history to /prompt_history folder + print("\nupdate: prompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + + prompter.save_state() + + # Generate CSV report for easy Human review in Excel + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + + print("\nupdate: csv output for human review - ", csv_output) + + return 0 + + +if __name__ == "__main__": + + invoice_processing(run_on_cpu=True) +``` + +# Document Summarizer + +```python + +""" This Example shows a packaged 'document_summarizer' prompt using the slim-summary-tool. It shows a variety of +techniques to summarize documents generally larger than a LLM context window, and how to assemble multiple source +batches from the document, as well as using a 'query' and 'topic' to focus on specific segments of the document. """ + +import os + +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def test_summarize_document(example="jd salinger"): + + # pull a sample document (or substitute a file_path and file_name of your own) + sample_files_path = Setup().load_sample_files(over_write=False) + + topic = None + query = None + fp = None + fn = None + + if example not in ["jd salinger", "employment terms", "just the comp", "un resolutions"]: + print ("not found example") + return [] + + if example == "jd salinger": + fp = os.path.join(sample_files_path, "SmallLibrary") + fn = "Jd-Salinger-Biography.docx" + topic = "jd salinger" + query = None + + if example == "employment terms": + fp = os.path.join(sample_files_path, "Agreements") + fn = "Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + topic = "executive compensation terms" + query = None + + if example == "just the comp": + fp = os.path.join(sample_files_path, "Agreements") + fn = "Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + topic = "executive compensation terms" + query = "base salary" + + if example == "un resolutions": + fp = os.path.join(sample_files_path, "SmallLibrary") + fn = "N2126108.pdf" + # fn = "N2137825.pdf" + topic = "key points" + query = None + + # optional parameters: 'query' - will select among blocks with the query term + # 'topic' - will pass a topic/issue as the parameter to the model to 'focus' the summary + # 'max_batch_cap' - caps the number of batches sent to the model + # 'text_only' - returns just the summary text aggregated + + kp = Prompt().summarize_document_fc(fp, fn, topic=topic, query=query, text_only=True, max_batch_cap=15) + + print(f"\nDocument summary completed - {len(kp)} Points") + for i, points in enumerate(kp): + print(i, points) + + return 0 + + +if __name__ == "__main__": + + print(f"\nExample: Summarize Documents\n") + + # 4 examples - ["jd salinger", "employment terms", "just the comp", "un resolutions"] + # -- "jd salinger" - summarizes key points about jd salinger from short biography document + # -- "employment terms" - summarizes the executive compensation terms across 15 page document + # -- "just the comp" - queries to find subset of document and then summarizes the key terms + # -- "un resolutions" - summarizes the un resolutions document + + summary_direct = test_summarize_document(example="employment terms") +``` + +For more examples, see the [prompt examples]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Prompts/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/examples/retrieval.md b/docs/examples/retrieval.md new file mode 100644 index 0000000..acf9158 --- /dev/null +++ b/docs/examples/retrieval.md @@ -0,0 +1,167 @@ +--- +layout: default +title: Retrieval +parent: Examples +nav_order: 7 +description: overview of the major modules and classes of LLMWare +permalink: /examples/retrieval +--- +# Retrieval - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + +# SEMANTIC Retrieval Example + +```python + +""" +This 'getting started' example demonstrates how to use basic semantic retrieval with the Query class + 1. Create a sample library + 2. Run a basic semantic query + 3. View the results +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def create_fin_docs_sample_library(library_name): + + print(f"update: creating library - {library_name}") + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "FinDocs") + parsing_output = library.add_files(ingestion_folder_path) + + print(f"update: building embeddings - may take a few minutes the first time") + + # note: if you have installed Milvus or another vector DB, please feel free to substitute + # note: if you have any memory constraints on laptop: + # (1) reduce embedding batch_size or ... + # (2) substitute "mini-lm-sbert" as embedding model + + library.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="chromadb",batch_size=200) + + return library + + +def basic_semantic_retrieval_example (library): + + # Create a Query instance + q = Query(library) + + # Set the keys that should be returned - optional - full set of keys will be returned by default + q.query_result_return_keys = ["distance","file_source", "page_num", "text"] + + # perform a simple query + my_query = "ESG initiatives" + query_results1 = q.semantic_query(my_query, result_count=20) + + # Iterate through query_results, which is a list of result dicts + print(f"\nQuery 1 - {my_query}") + for i, result in enumerate(query_results1): + print("results - ", i, result) + + # perform another query + my_query2 = "stock performance" + query_results2 = q.semantic_query(my_query2, result_count=10) + + print(f"\nQuery 2 - {my_query2}") + for i, result in enumerate(query_results2): + print("results - ", i, result) + + # perform another query + my_query3 = "cloud computing" + + # note: use of embedding_distance_threshold will cap results with distance < 1.0 + query_results3 = q.semantic_query(my_query3, result_count=50, embedding_distance_threshold=1.0) + + print(f"\nQuery 3 - {my_query3}") + for i, result in enumerate(query_results3): + print("result - ", i, result) + + return [query_results1, query_results2, query_results3] + + +if __name__ == "__main__": + + print(f"Example - Running a Basic Semantic Query") + + LLMWareConfig().set_active_db("sqlite") + + # step 1- will create library + embeddings with Financial Docs + lib = create_fin_docs_sample_library("lib_semantic_query_1") + + # step 2- run query against the library and embeddings + my_results = basic_semantic_retrieval_example(lib) +``` + +For more examples, see the [retrieval examples]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Retrieval/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/examples/structured_tables.md b/docs/examples/structured_tables.md new file mode 100644 index 0000000..ce224f2 --- /dev/null +++ b/docs/examples/structured_tables.md @@ -0,0 +1,225 @@ +--- +layout: default +title: Structured Tables +parent: Examples +nav_order: 9 +description: overview of the major modules and classes of LLMWare +permalink: /examples/structured_tables +--- +# Structured Tables - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + +```python + +""" This example shows the basic recipe for creating a CustomTable with LLMWare and a few of the basic methods + to quickly get started. + + In this example, we will build a very simple 'hello world' Files table, which we will build upon in a future + example by aggregating a more interesting and useful set of attributes from a LLMWare Library collection. + + CustomTable is designed to work with the text collection databases supported by LLMWare: + + SQL DBs --- Postgres and SQLIte + NoSQL DB --- Mongo DB + + Even though Mongo does not require a schema for inserting and retrieving information, the CustomTable method + will expect a defined schema to be provided (good best practice, in any case). """ + +from llmware.resources import CustomTable + + +def hello_world_custom_table(): + + # simple schema for a table to track Files/Documents + # note: the schema is a python dictionary, with named keys, and the value corresponding to the data type + # for sqlite and postgres, any standard sql data type should generally work + + files_schema = {"custom_doc_num": "integer", + "file_name": "text", + "comments": "text"} + + # create a CustomTable object + db_name = "sqlite" + table_name = "files_table_1000" + ct = CustomTable(db=db_name,table_name=table_name, schema=files_schema) + + # insert a few sample rows - each row is a dictionary with keys from the schema, and the *actual* values + r1 = {"custom_doc_num": 1, "file_name": "technical_manual.pdf", "comments": "very useful overview"} + ct.write_new_record(r1) + + r2 = {"custom_doc_num": 2, "file_name": "work_presentation.pptx", "comments": "need to save for future reference"} + ct.write_new_record(r2) + + r3 = {"custom_doc_num": 3, "file_name": "dataset.json", "comments": "will use in next project"} + ct.write_new_record(r3) + + # to see the entries - pull all items from the table + all_results = ct.get_all() + + print("\nTEST #1 - Retrieving All Elements") + for i, res in enumerate(all_results): + print("results: ", i, res) + + # look at the database schema + schema = ct.get_schema() + + print("\nTEST #2 - Getting the Table Schema") + print("schema: ", schema) + + schema_str = ct.sql_table_create_string() + + print("table create sql: ", schema_str) + + # perform a basic lookup with 'key' and 'value' + f = ct.lookup("custom_doc_num", 2) + + print("\nTEST #3 - Basic Lookup - 'custom_doc_num' = 2") + print("lookup: ", f) + + # if you prefer SQL, pass a SQL query directly (note: this will only work on Postgres and SQLite) + + if db_name == "sqlite": + + # note: our standard 'unpacking' of a row of sqlite includes the rowid attribute + custom_query = f"SELECT rowid, * FROM {table_name} WHERE custom_doc_num = 3;" + + elif db_name == "postgres": + custom_query = f"SELECT * FROM {table_name} WHERE custom_doc_num = 3;" + + elif db_name == "mongo": + custom_query = {"custom_doc_num": 3} + else: + print("must use either sqlite, postgres or mongo") + return -1 + + cf = ct.custom_lookup(custom_query) + + print("\nTEST #4 - Custom SQL Lookup - 'custom_doc_num' = 3") + print("custom query lookup: ", cf) + + print("\nTEST #5 - Making Updates and Deletes") + + # to delete a record + ct.delete_record("custom_doc_num", 1) + print("deleted record") + + # to update the values of a record + ct.update_record({"custom_doc_num": 2}, "file_name", "work_presentation_update_v2.pptx") + print("updated record") + + updated_all_results = ct.get_all() + + for i, res in enumerate(updated_all_results): + print("updated results: ", i, res) + + print("\nTEST #6 - Delete Table - uncomment and set confirm=True") + # done? delete the table and start over + # -- note: confirm=True must be set + # ct.delete_table(confirm=False) + + # look at all tables in the database + tables = ct.list_all_tables() + + print("\nTEST #7 - View all of the tables on the DB") + for i, t in enumerate(tables): + print("tables:" ,i, t) + + return 0 + + +if __name__ == "__main__": + + hello_world_custom_table() +``` + + +These examples illustrate the use of the CustomTable class to quickly create SQL tables that can be used in conjunction with LLM-based workflows. + +1. [**Intro to CustomTables**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Structured_Tables/create_custom_table-1.py) + + - Getting started with using CustomTables + +2. [**Loading CSV into CustomTables**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Structured_Tables/loading_csv_into_custom_table-2a.py) + + - Loading CSV into CustomTables + +3. [**Loading CSV into Library (Configured)**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Structured_Tables/loading_csv_w_config_options-2b.py) + + - Loading CSV into Library + +4. [**Loading JSON into CustomTables**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Stuctured_Tables/loading_json_custom_table-3a.py) + + - Loading JSON into CustomTable database + +5 [**Loading JSON into Library (Configured)**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Stuctured_Tables/loading_json_w_config_options-3b.py) + + - Loading JSON into a library with configuration + + +For more examples, see the [structured tables example]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Structured_Tables/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + + diff --git a/docs/examples/ui.md b/docs/examples/ui.md new file mode 100644 index 0000000..5e67940 --- /dev/null +++ b/docs/examples/ui.md @@ -0,0 +1,101 @@ +--- +layout: default +title: UI +parent: Examples +nav_order: 8 +description: overview of the major modules and classes of LLMWare +permalink: /examples/ui +--- +# UI - Introduction by Examples +We introduce ``llmware`` through self-contained examples. + +**UI Scenarios** + +We provide several 'UI' examples that show how to use LLMWare in a complex recipe combining different elements to accomplish a specific objective. While each example is still high-level, it is shared in the spirit of providing a high-level framework 'starting point' that can be developed in more detail for a variety of common use cases. All of these examples use small, specialized models, running locally - 'Small, but Mighty' ! + + +1. [**GGUF Streaming Chatbot**](https://www.github.com/llmware-ai/llmware/tree/main/examples/UI/gguf_streaming_chatbot.py) + + - Locally deployed chatbot using leading open source chat models, including Phi-3-GGUF + - Uses Streamlit + - Core simple framework of ~20 lines using llmware and Streamlit + +2. [**Simple RAG UI with Streamlit**](https://www.github.com/llmware-ai/llmware/tree/main/examples/UI/simple_rag_ui_with_streamlit.py) + + - Simple RAG UI + +3. [**RAG UI with Query Topic with Streamlit**](https://www.github.com/llmware-ai/llmware/tree/main/examples/UI/rag_ui_with_query_topic_with_streamlit.py) + + - UI demonstrating UI with query topic in RAG scenario + +4. [**Using Streamlit Chat UI**](https://www.github.com/llmware-ai/llmware/tree/main/examples/UI/using_streamlit_chat_ui.py) + + - Basic Streamlit Chat UI + + +For more examples, see the [UI examples]((https://www.github.com/llmware-ai/llmware/tree/main/examples/UI/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/examples/use_cases.md b/docs/examples/use_cases.md new file mode 100644 index 0000000..c96bdc2 --- /dev/null +++ b/docs/examples/use_cases.md @@ -0,0 +1,137 @@ +--- +layout: default +title: Use Cases +parent: Examples +nav_order: 1 +description: overview of the major modules and classes of LLMWare +permalink: /examples/use_cases +--- +🚀 Use Cases Examples 🚀 +--- + +**End-to-End Scenarios** + +We provide several 'end-to-end' examples that show how to use LLMWare in a complex recipe combining different elements to accomplish a specific objective. While each example is still high-level, it is shared in the spirit of providing a high-level framework 'starting point' that can be developed in more detail for a variety of common use cases. All of these examples use small, specialized models, running locally - 'Small, but Mighty' ! + + +1. [**Research Automation with Agents and Web Services**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/web_services_slim_fx.py) + + - Prepare a 30-key research analysis on a company + - Extract key lookup and other information from an earnings press release + - Automatically use the lookup data for real-time stock information from YFinance + - Automatically use the lookup date for background company history information in Wikipedia + - Run LLM prompts to ask key questions of the Wikipedia sources + - Aggregate into a consolidated research analysis + - All with local open source models + + +2. [**Invoice Processing**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/invoice_processing.py) + + - Parse a batch of invoices (provided as sample files) + - Extract key information from the invoices + - Save the prompt state for follow-up review and analysis + + +3. [**Analyzing and Extracting Voice Transcripts**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/parsing_great_speeches.py) + + - Voice transcription of 50+ wav files of great speeches of the 20th century + - Run text queries against the transcribed wav files + - Execute LLM agent inferences to extract and identify key elements of interest + - Prepare 'bibliography' with the key extracted points, including time-stamp + + +4. [**MSA Processing**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/msa_processing.py) + + - Identify the termination provisions in Master Service Agreements among a larger batch of contracts + - Parse and query a large batch of contracts and identify the agreements with "Master Service Agreement" on the first page + - Find the termination provisions in each MSA + - Prompt LLM to read the termination provisions and answer a key question + - Run a fact-check and source-check on the LLM response + - Save all of the responses in CSV and JSON for follow-up review. + + +5. [**Querying a CSV**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/agent_with_custom_tables.py) + + - Start running natural language queries on CSVs with Postgres and slim-sql-tool. + - Load a sample 'customer_table.csv' into Postgres + - Start running natural language queries that get converted into SQL and query the DB + + +6. [**Contract Analysis**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/contract_analysis_on_laptop_with_bling_models.py) + + - Extract key information from set of employment agreement + - Use a simple retrieval strategy with keyword search to identify key provisions and topic areas + - Prompt LLM to read the key provisions and answer questions based on those source materials + +7. [**Slicing and Dicing Office Docs**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/slicing_and_dicing_office_docs.py) + + - Shows a variety of advanced parsing techniques with Office document formats packaged in ZIP archives + - Extracts tables and images, runs OCR against the embedded images, exports the whole library, and creates dataset + + +For more examples, see the [use cases example]((https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/) in the main repo. + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/getting_started/clone_repo.md b/docs/getting_started/clone_repo.md new file mode 100644 index 0000000..a2649d6 --- /dev/null +++ b/docs/getting_started/clone_repo.md @@ -0,0 +1,92 @@ +--- +layout: default +title: Clone Repo +parent: Getting Started +nav_order: 3 +permalink: /getting_started/clone_repo +--- + +## ✍️ Working with the llmware Github repository + +The llmware repo can be pulled locally to get access to all the examples, or to work directly with the latest version of the llmware code. + +```bash +git clone git@github.com:llmware-ai/llmware.git +``` + +We have provided a **welcome_to_llmware** automation script in the root of the repository folder. After cloning: +- On Windows command line: `.\welcome_to_llmware_windows.sh` +- On Mac / Linux command line: `sh ./welcome_to_llmware.sh` + +Alternatively, if you prefer to complete setup without the welcome automation script, then the next steps include: + +1. **install requirements.txt** - inside the /llmware path - e.g., ```pip3 install -r llmware/requirements.txt``` + +2. **install requirements_extras.txt** - inside the /llmware path - e.g., ```pip3 install -r llmware/requirements_extras.txt``` (Depending upon your use case, you may not need all or any of these installs, but some of these will be used in the examples.) + +3. **run examples** - copy one or more of the example .py files into the root project path. (We have seen several IDEs that will attempt to run interactively from the nested /example path, and then not have access to the /llmware module - the easy fix is to just copy the example you want to run into the root path). + +4. **install vector db** - no-install vector db options include milvus lite, chromadb, faiss and lancedb - which do not require a server install, but do require that you install the python sdk library for that vector db, e.g., `pip3 install pymilvus`, or `pip3 install chromadb`. If you look in [examples/Embedding](https://github.com/llmware-ai/llmware/tree/main/examples/Embedding), you will see examples for getting started with various vector DB, and in the root of the repo, you will see easy-to-get-started docker compose scripts for installing milvus, postgres/pgvector, mongo, qdrant, neo4j, and redis. + +5. Note: we have seen recently issues with Pytorch==2.3 on some platforms - if you run into any issues, we have seen that uninstalling Pytorch and downleveling to Pytorch==2.1 usually solves the problem. + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/getting_started/fast_start.md b/docs/getting_started/fast_start.md new file mode 100644 index 0000000..905c939 --- /dev/null +++ b/docs/getting_started/fast_start.md @@ -0,0 +1,143 @@ +--- +layout: default +title: Fast Start +parent: Getting Started +nav_order: 4 +permalink: /getting_started/fast_start +--- + +Fast Start: Learning RAG with llmware through 6 examples +--- + +**Welcome to llmware!** + +Fast Start is a structured series of 6 self-contained examples and accompanying videos that walk through the core foundational components of RAG with LLMWare. +Set up + +`pip3 install llmware` or, if you prefer clone the github repo locally, e.g., `git clone git@github.com:llmware-ai/llmware.git +`. + +Platforms: +- Mac M1/M2/M3, Windows, Linux (Ubuntu 20 or Ubuntu 22 preferred) +- RAM: 16 GB minimum +- Python 3.9, 3.10, 3.11, 3.12 +- Pull the latest version of llmware == 0.2.14 (as of mid-May 2024) +- Please note that we have updated the examples from the original versions, to use new features in llmware, so there may be minor differences with the videos, which are annotated in the comments in each example. + +There are 6 examples, designed to be used step-by-step, but each is self-contained, +so you can feel free to jump into any of the examples, in any order, that you prefer. + +Each example has been designed to be "copy-paste" and RUN with lots of helpful comments and explanations embedded in the code samples. + +Please check out our [Fast Start Youtube tutorials](https://www.youtube.com/playlist?list=PL1-dn33KwsmD7SB9iSO6vx4ZLRAWea1DB) that walk through each example below. + +Examples: + +**Section I - Learning the Main Components** +1. **Library** - parse, text chunk, and index to convert a "pile of files" into an AI-ready knowledge-base. [Video](https://youtu.be/2xDefZ4oBOM?si=8vRCvqj0-HG3zc4c) + +2. **Embeddings** - apply an embedding model to the Library, store vectors, and start enabling natural language queries. [Video](https://youtu.be/xQEk6ohvfV0?si=B3X25ZsAZfW4AR_3) + +3. **Prompts** & **Model Catalog** - start running inferences and building prompts. [Video](https://youtu.be/swiu4oBVfbA?si=0IVmLhiiYS3-pMIg) + +**Section II - Connecting Knowledge with Prompts - 3 scenarios** + +4. **RAG with Text Query** - start integrating documents into prompts. [Video](https://youtu.be/6oALi67HP7U?si=pAbvio4ULXTIXKdL) + +5. **RAG with Semantic Query** - use natural language queries on documents and integrate with prompts. [Video](https://youtu.be/XT4kIXA9H3Q?si=EBCAxVXBt5vgYY8s) + +6. **RAG with more complex retrieval** - start integrating more complex retrieval patterns. [Video](https://youtu.be/G1Q6Ar8THbo?si=vIVAv35uXAcnaUJy) + +After completing these 6 examples, you should have a good foundation and set of recipes to start +exploring the other 100+ examples in the /examples folder, and build more sophisticated +LLM-based applications. + +**Models** + - All of these examples are optimized for using local CPU-based models, primarily BLING and DRAGON. + - If you want to substitute for any other model in the catalog, it is generally as easy as + switching the model_name. If the model requires API keys, we show in the examples how to pass those keys as an + environment variable. + +**Collection Databases** + - Our parsers are optimized to index text chunks directly into a persistent data store. + - For Fast Start, we will use "sqlite" which is an embedded database, requiring no install + - For more scalable deployment, we would recommend either "mongo" or "postgres" + - Install instructions for "mongo" and "postgres" are provided in docker-compose files in the repository + +**Vector Databases** + - For Fast Start, we will use "chromadb" in persistent 'file' mode, requiring no install. + - Note: if you are using Python < 3.12, then please feel free to substitute for faiss (which was used in the videos). + - Note: depending upon how and when you installed llmware, you may need to `pip install chromadb`. + - For more scalable deployment, we would recommend installing one of 9 supported vector databases, + including Milvus, PGVector (Postgres), Redis, Qdrant, Neo4j, Mongo-Atlas, Chroma, LanceDB, or Pinecone. + - Install instructions provided in "examples/Embedding" for specific db, as well as docker-compose scripts. + +**Local Private** + - All of the processing will take place locally on your laptop. + +*This is an ongoing initiative to provide easy-to-get-started tutorials - we welcome and encourage feedback, as well +as contributions with examples and other tips for helping others on their LLM application journeys!* + +**Let's get started!** + + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/getting_started/getting_started.md b/docs/getting_started/getting_started.md new file mode 100644 index 0000000..db0d717 --- /dev/null +++ b/docs/getting_started/getting_started.md @@ -0,0 +1,166 @@ +--- +layout: default +title: Getting Started +nav_order: 2 +has_children: true +description: getting started with llmware +permalink: /getting_started +--- + +## Welcome to +
    +
  • + llmware +
    • +
+ +## 🧰🛠️🔩The Ultimate Toolkit for Enterprise RAG Pipelines with Small, Specialized Models + +From quickly building POCs to scalable LLM Apps for the enterprise, LLMWare is packed with all the tools you need. + +`llmware` is an integrated framework with over 50+ small, specialized, open source models for quickly developing LLM-based applications including Retrieval Augmented Generation (RAG) and Multi-Step Orchestration of Agent Workflows. + +This project provides a comprehensive set of tools that anyone can use - from a beginner to the most sophisticated AI developer - to rapidly build industrial-grade, knowledge-based enterprise LLM applications. + +Our specific focus is on making it easy to integrate open source small specialized models and connecting enterprise knowledge safely and securely. + + +## Getting Started + +1. Install llmware - `pip3 install llmware` + + +2. Make sure that you are running on a [supported platform](https://github.com/llmware-ai/llmware/blob/main/docs/getting_started/platforms.md#platform-support). + + +3. Learn by example: + + -- [Fast Start examples](https://www.github.com/llmware-ai/llmware/tree/main/fast_start) - structured set of 6 examples (with no DB installations required) to learn the main concepts of RAG with LLMWare - each example has extensive comments, and a supporting video on Youtube to walk you through it. + + -- [Getting Started examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started) - heavily-annotated examples that review many getting started elements - selecting a database, loading sample files, working with libraries, and how to use the Model Catalog. + + -- [Use Case examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases) - longer examples that integrate several components of LLMWare to provide a framework for a solution for common use case patterns. + + -- Dive into specific area of interest - [Parsing](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing) - [Models](https://www.github.com/llmware-ai/llmware/tree/main/examples/Models) - [Prompts](https://www.github.com/llmware-ai/llmware/tree/main/examples/Models) - [Agents](https://www.github.com/llmware-ai/llmware/tree/main/examples/SLIM-Agents) - and many more ... + + +4. We provide extensive [sample files](https://www.github.com/llmware-ai/tree/main/examples/Getting_Started/loading_sample_files.py) integrated into the examples, so you can copy-paste-run, and quickly validate that the installation is set up correctly, and to start seeing key classes and methods in action. We would encourage you to start with the 'out of the box' example first, and then use the example as the launching point for inserting your documents, models, queries, and workflows. + + +5. Learn by watching: check out the [LLMWare Youtube channel](https://www.youtube.com/@llmware). + + +6. Share with the community: join us on [Discord](https://discord.gg/MhZn5Nc39h). + + +[Install llmware](#install-llmware){: .btn .btn-primary .fs-5 .mb-4 .mb-md-0 .mr-2 } +[Common Setup & Configuration Items](#platform-support){: .btn .fs-5 .mb-4 .mb-md-0 } +[Architecture](architecture.md/#llmware-architecture){: .btn .fs-5 .mb-4 .mb-md-0 } +[View llmware on GitHub](https://www.github.com/llmware-ai/llmware/tree/main){: .btn .fs-5 .mb-4 .mb-md-0 } +[Open an Issue on GitHub](https://www.github.com/llmware-ai/llmware/issues){: .btn .fs-5 .mb-4 .mb-md-0 } + + + +# Install llmware + +___ +**Using Pip Install** + +- Installing llmware is easy: `pip3 install llmware` + + +- If you prefer, we also provide a set of recent wheels in the [wheel archives](https://www.github.com/llmware-ai/llmware/tree/main/wheel_archives) in this repository, which can be downloaded individually and used as follows: + +```bash +pip3 install llmware-0.2.12-py3-none-any.wheel +```` + +- We generally keep the main branch of this repository current with all changes, but we only publish new wheels to PyPi approximately once per week + +___ + +___ +**Cloning the Repository** + +- If you prefer to clone the repository: + +```bash +git clone git@github.com:llmware-ai/llmware.git +``` + +- The llmware package is contained entirely in the /llmware folder path, so you should be able to drop this folder (with all of its contents) into a project tree, and use the llmware module essentially the same as a pip install. + +- Please ensure that you are capturing and updating the /llmware/lib folder, which includes required compiled shared libraries. If you prefer, you can keep only those libs required for your OS platform. + +- After cloning the repo, we provide a short 'welcome to llmware' automation script, which can be used to install the projects requirements (from llmware/requirements.txt), install several optional dependencies that are commonly used in examples, copy several good 'getting started' examples into the root folder, and then run a 'welcome_example.py' script to get started using our models. To use the "welcome to llmware" script: + +Windows: +```bash +.\welcome_to_llmware_windows.sh +``` + +Mac/Linux: +```bash +sh ./welcome_to_llmware.sh +``` + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/getting_started/installation.md b/docs/getting_started/installation.md new file mode 100644 index 0000000..87304d6 --- /dev/null +++ b/docs/getting_started/installation.md @@ -0,0 +1,119 @@ +--- +layout: default +title: Installation +parent: Getting Started +nav_order: 2 +permalink: /getting_started/installation +--- + +## Installation + +Set up + +`pip3 install llmware` or, if you prefer clone the github repo locally, e.g., `git clone git@github.com:llmware-ai/llmware.git +`. + +Platforms: +- Mac M1/M2/M3, Windows, Linux (Ubuntu 20 or Ubuntu 22 preferred) +- RAM: 16 GB minimum +- Python 3.9, 3.10, 3.11 (note: not supported on 3.12 - coming soon!) +- Pull the latest version of llmware == 0.2.11 (as of end of April 2024) +- Please note that we have updated the examples from the original versions, to use new features in llmware, so there may be minor differences with the videos, which are annotated in the comments in each example. + + +## Wheel Archive + +- If you prefer, we also provide a set of recent wheels in the [wheel archives](https://www.github.com/llmware-ai/llmware/tree/main/wheel_archives) in this repository, which can be downloaded individually and used as follows: + +```bash +pip3 install llmware-0.2.12-py3-none-any.wheel +```` + +- We generally keep the main branch of this repository current with all changes, but we only publish new wheels to PyPi approximately once per week + +___ + +___ +**Cloning the Repository** + +- If you prefer to clone the repository: + +```bash +git clone git@github.com:llmware-ai/llmware.git +``` + +- The llmware package is contained entirely in the /llmware folder path, so you should be able to drop this folder (with all of its contents) into a project tree, and use the llmware module essentially the same as a pip install. + +- Please ensure that you are capturing and updating the /llmware/lib folder, which includes required compiled shared libraries. If you prefer, you can keep only those libs required for your OS platform. + +- After cloning the repo, we provide a short 'welcome to llmware' automation script, which can be used to install the projects requirements (from llmware/requirements.txt), install several optional dependencies that are commonly used in examples, copy several good 'getting started' examples into the root folder, and then run a 'welcome_example.py' script to get started using our models. To use the "welcome to llmware" script: + +Windows: +```bash +.\welcome_to_llmware_windows.sh +``` + +Mac/Linux: +```bash +sh ./welcome_to_llmware.sh +``` + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/getting_started/overview.md b/docs/getting_started/overview.md new file mode 100644 index 0000000..be82cc8 --- /dev/null +++ b/docs/getting_started/overview.md @@ -0,0 +1,653 @@ +--- +layout: default +title: Overview +parent: Getting Started +nav_order: 1 +permalink: /getting_started/overview +--- + +## Welcome to +
    +
  • + llmware +
    • +
+ +## 🧰🛠️🔩Building Enterprise RAG Pipelines with Small, Specialized Models + +`llmware` provides a unified framework for building LLM-based applications (e.g, RAG, Agents), using small, specialized models that can be deployed privately, integrated with enterprise knowledge sources safely and securely, and cost-effectively tuned and adapted for any business process. + + `llmware` has two main components: + + 1. **RAG Pipeline** - integrated components for the full lifecycle of connecting knowledge sources to generative AI models; and + + 2. **50+ small, specialized models** fine-tuned for key tasks in enterprise process automation, including fact-based question-answering, classification, summarization, and extraction. + +By bringing together both of these components, along with integrating leading open source models and underlying technologies, `llmware` offers a comprehensive set of tools to rapidly build knowledge-based enterprise LLM applications. + +Most of our examples can be run without a GPU server - get started right away on your laptop. + +## 🎯 Key features +Writing code with`llmware` is based on a few main concepts: + +
+Model Catalog: Access all models the same way with easy lookup, regardless of underlying implementation. + + + +```python +# 150+ Models in Catalog with 50+ RAG-optimized BLING, DRAGON and Industry BERT models +# Full support for GGUF, HuggingFace, Sentence Transformers and major API-based models +# Easy to extend to add custom models - see examples + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# all models accessed through the ModelCatalog +models = ModelCatalog().list_all_models() + +# to use any model in the ModelCatalog - "load_model" method and pass the model_name parameter +my_model = ModelCatalog().load_model("llmware/bling-phi-3-gguf") +output = my_model.inference("what is the future of AI?", add_context="Here is the article to read") + +# to integrate model into a Prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") +response = prompter.prompt_main("what is the future of AI?", context="Insert Sources of information") +``` + +
+ +
+Library: ingest, organize and index a collection of knowledge at scale - Parse, Text Chunk and Embed. + +```python + +from llmware.library import Library + +# to parse and text chunk a set of documents (pdf, pptx, docx, xlsx, txt, csv, md, json/jsonl, wav, png, jpg, html) + +# step 1 - create a library, which is the 'knowledge-base container' construct +# - libraries have both text collection (DB) resources, and file resources (e.g., llmware_data/accounts/{library_name}) +# - embeddings and queries are run against a library + +lib = Library().create_new_library("my_library") + +# step 2 - add_files is the universal ingestion function - point it at a local file folder with mixed file types +# - files will be routed by file extension to the correct parser, parsed, text chunked and indexed in text collection DB + +lib.add_files("/folder/path/to/my/files") + +# to install an embedding on a library - pick an embedding model and vector_db +lib.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="milvus", batch_size=500) + +# to add a second embedding to the same library (mix-and-match models + vector db) +lib.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="chromadb", batch_size=100) + +# easy to create multiple libraries for different projects and groups + +finance_lib = Library().create_new_library("finance_q4_2023") +finance_lib.add_files("/finance_folder/") + +hr_lib = Library().create_new_library("hr_policies") +hr_lib.add_files("/hr_folder/") + +# pull library card with key metadata - documents, text chunks, images, tables, embedding record +lib_card = Library().get_library_card("my_library") + +# see all libraries +all_my_libs = Library().get_all_library_cards() + +``` +
+ +
+Query: query libraries with mix of text, semantic, hybrid, metadata, and custom filters. + +```python + +from llmware.retrieval import Query +from llmware.library import Library + +# step 1 - load the previously created library +lib = Library().load_library("my_library") + +# step 2 - create a query object and pass the library +q = Query(lib) + +# step 3 - run lots of different queries (many other options in the examples) + +# basic text query +results1 = q.text_query("text query", result_count=20, exact_mode=False) + +# semantic query +results2 = q.semantic_query("semantic query", result_count=10) + +# combining a text query restricted to only certain documents in the library and "exact" match to the query +results3 = q.text_query_with_document_filter("new query", {"file_name": "selected file name"}, exact_mode=True) + +# to apply a specific embedding (if multiple on library), pass the names when creating the query object +q2 = Query(lib, embedding_model_name="mini_lm_sbert", vector_db="milvus") +results4 = q2.semantic_query("new semantic query") +``` + +
+ +
+Prompt with Sources: the easiest way to combine knowledge retrieval with a LLM inference. + +```python + +from llmware.prompts import Prompt +from llmware.retrieval import Query +from llmware.library import Library + +# build a prompt +prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") + +# add a file -> file is parsed, text chunked, filtered by query, and then packaged as model-ready context, +# including in batches, if needed, to fit the model context window + +source = prompter.add_source_document("/folder/to/one/doc/", "filename", query="fast query") + +# attach query results (from a Query) into a Prompt +my_lib = Library().load_library("my_library") +results = Query(my_lib).query("my query") +source2 = prompter.add_source_query_results(results) + +# run a new query against a library and load directly into a prompt +source3 = prompter.add_source_new_query(my_lib, query="my new query", query_type="semantic", result_count=15) + +# to run inference with 'prompt with sources' +responses = prompter.prompt_with_source("my query") + +# to run fact-checks - post inference +fact_check = prompter.evidence_check_sources(responses) + +# to view source materials (batched 'model-ready' and attached to prompt) +source_materials = prompter.review_sources_summary() + +# to see the full prompt history +prompt_history = prompter.get_current_history() +``` + +
+ +
+RAG-Optimized Models - 1-7B parameter models designed for RAG workflow integration and running locally. + +``` +""" This 'Hello World' example demonstrates how to get started using local BLING models with provided context, using both +Pytorch and GGUF versions. """ + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +# this is the main script to be run + +def bling_meets_llmware_hello_world (model_name): + + t0 = time.time() + + # load the questions + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + # load the model + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"] + , prompt_name="default_with_context",temperature=0.30) + + # print out the results + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # list of 'rag-instruct' laptop-ready small bling models on HuggingFace + + pytorch_models = ["llmware/bling-1b-0.1", # most popular + "llmware/bling-tiny-llama-v0", # fastest + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3" # most accurate (and newest) + ] + + # Quantized GGUF versions generally load faster and run nicely on a laptop with at least 16 GB of RAM + gguf_models = ["bling-phi-3-gguf", "bling-stablelm-3b-tool", "dragon-llama-answer-tool", "dragon-yi-answer-tool", "dragon-mistral-answer-tool"] + + # try model from either pytorch or gguf model list + # the newest (and most accurate) is 'bling-phi-3-gguf' + + bling_meets_llmware_hello_world(gguf_models[0] + + # check out the model card on Huggingface for RAG benchmark test performance results and other useful information +``` + +
+ +
+Simple-to-Scale Database Options - integrated data stores from laptop to parallelized cluster. + +```python + +from llmware.configs import LLMWareConfig + +# to set the collection database - mongo, sqlite, postgres +LLMWareConfig().set_active_db("mongo") + +# to set the vector database (or declare when installing) +# --options: milvus, pg_vector (postgres), redis, qdrant, faiss, pinecone, mongo atlas +LLMWareConfig().set_vector_db("milvus") + +# for fast start - no installations required +LLMWareConfig().set_active_db("sqlite") +LLMWareConfig().set_vector_db("chromadb") # try also faiss and lancedb + +# for single postgres deployment +LLMWareConfig().set_active_db("postgres") +LLMWareConfig().set_vector_db("postgres") + +# to install mongo, milvus, postgres - see the docker-compose scripts as well as examples + +``` + +
+ +
+ + 🔥 Agents with Function Calls and SLIM Models 🔥 + +```python + +from llmware.agents import LLMfx + +text = ("Tesla stock fell 8% in premarket trading after reporting fourth-quarter revenue and profit that " + "missed analysts’ estimates. The electric vehicle company also warned that vehicle volume growth in " + "2024 'may be notably lower' than last year’s growth rate. Automotive revenue, meanwhile, increased " + "just 1% from a year earlier, partly because the EVs were selling for less than they had in the past. " + "Tesla implemented steep price cuts in the second half of the year around the world. In a Wednesday " + "presentation, the company warned investors that it’s 'currently between two major growth waves.'") + +# create an agent using LLMfx class +agent = LLMfx() + +# load text to process +agent.load_work(text) + +# load 'models' as 'tools' to be used in analysis process +agent.load_tool("sentiment") +agent.load_tool("extract") +agent.load_tool("topics") +agent.load_tool("boolean") + +# run function calls using different tools +agent.sentiment() +agent.topics() +agent.extract(params=["company"]) +agent.extract(params=["automotive revenue growth"]) +agent.xsum() +agent.boolean(params=["is 2024 growth expected to be strong? (explain)"]) + +# at end of processing, show the report that was automatically aggregated by key +report = agent.show_report() + +# displays a summary of the activity in the process +activity_summary = agent.activity_summary() + +# list of the responses gathered +for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + +output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + +``` + +
+
+ + 🚀 Start coding - Quick Start for RAG 🚀 + +```python +# This example illustrates a simple contract analysis +# using a RAG-optimized LLM running locally + +import os +import re +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + +def contract_analysis_on_laptop (model_name): + + # In this scenario, we will: + # -- download a set of sample contract files + # -- create a Prompt and load a BLING LLM model + # -- parse each contract, extract the relevant passages, and pass questions to a local LLM + + # Main loop - Iterate thru each contract: + # + # 1. parse the document in memory (convert from PDF file into text chunks with metadata) + # 2. filter the parsed text chunks with a "topic" (e.g., "governing law") to extract relevant passages + # 3. package and assemble the text chunks into a model-ready context + # 4. ask three key questions for each contract to the LLM + # 5. print to the screen + # 6. save the results in both json and csv for furthe processing and review. + + # Load the llmware sample files + + print (f"\n > Loading the llmware sample files...") + + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path,"Agreements") + + # Query list - these are the 3 main topics and questions that we would like the LLM to analyze for each contract + + query_list = {"executive employment agreement": "What are the name of the two parties?", + "base salary": "What is the executive's base salary?", + "vacation": "How many vacation days will the executive receive?"} + + # Load the selected model by name that was passed into the function + + print (f"\n > Loading model {model_name}...") + + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + + # Main loop + + for i, contract in enumerate(os.listdir(contracts_path)): + + # excluding Mac file artifact (annoying, but fact of life in demos) + if contract != ".DS_Store": + + print("\nAnalyzing contract: ", str(i+1), contract) + + print("LLM Responses:") + + for key, value in query_list.items(): + + # step 1 + 2 + 3 above - contract is parsed, text-chunked, filtered by topic key, + # ... and then packaged into the prompt + + source = prompter.add_source_document(contracts_path, contract, query=key) + + # step 4 above - calling the LLM with 'source' information already packaged into the prompt + + responses = prompter.prompt_with_source(value, prompt_name="default_with_context") + + # step 5 above - print out to screen + + for r, response in enumerate(responses): + print(key, ":", re.sub("[\n]"," ", response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + + # step 6 above - saving the analysis to jsonl and csv + + # Save jsonl report to jsonl to /prompt_history folder + print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + prompter.save_state() + + # Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + print("csv output saved at: ", csv_output) + + +if __name__ == "__main__": + + # use local cpu model - try the newest - RAG finetune of Phi-3 quantized and packaged in GGUF + model = "bling-phi-3-gguf" + + contract_analysis_on_laptop(model) + +``` +
+ + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/getting_started/platforms.md b/docs/getting_started/platforms.md new file mode 100644 index 0000000..7a225df --- /dev/null +++ b/docs/getting_started/platforms.md @@ -0,0 +1,155 @@ +--- +layout: default +title: Platforms Supported +parent: Getting Started +nav_order: 5 +permalink: /getting_started/platforms +--- +___ +# Platform Support +___ + +**Platform Supported** + +- **Python 3.9+** (note that we just added support for 3.12 starting in llmware version 0.2.12) + + +- **System RAM**: recommended 16 GB RAM minimum (to run most local models on CPU) + + +- **OS Supported**: Mac OS M1/M2/M3, Windows, Linux Ubuntu 20/22. We regularly build and test on Windows and Linux platforms with and without CUDA drivers. + + +- **Deprecated OS**: Linux Aarch64 (0.2.6) and Mac x86 (0.2.10) - most features of llmware should work on these platforms, but new features integrated since those versions will not be available. If you have a particular need to work on one of these platforms, please raise an Issue, and we can work with you to try to find a solution. + + +- **Linux**: we build to GLIBC 2.31+ - so Linux versions with older GLIBC drivers will generally not work (e.g., Ubuntu 18). To check the GLIBC version, you can use the command `ldd --version`. If it is 2.31 or any higher version, it should work. + +___ + +___ +**Database** + +- LLMWare is an enterprise-grade data pipeline designed for persistent storage of key artifacts throughout the pipeline. We provide several options to parse 'in-memory' and write to jsonl files, but most of the functionality of LLMWare assumes that a persistent scalable data store will be used. + + +- There are three different types of data storage used in LLMWare: + + 1. **Text Collection database** - all of the LLMWare parsers, by default, parse and text chunk unstructured content (and associated metadata) into one of three databases used for text collections, organized in Libraries - **MongoDB**, **Postgres** and **SQLite**. + + 2. **Vector database** - for storing and retrieving semantic embedding vectors, LLMWare supports the following vector databases - Milvus, PG Vector / Postgres, Qdrant, ChromaDB, Redis, Neo4J, Lance DB, Mongo-Atlas, Pinecone and FAISS. + + 3. **SQL Tables database** - for easily integrating table-based data into LLM workflows through the CustomTable class and for using in conjunction with a Text-2-SQL workflow - supported on Postgres and SQLite. + + +- **Fast Start** option: you can start using SQLite locally without any separate installation by setting `LLMWareConfig.set_active_db("sqlite")` as shown in [configure_db_example](https://www.github.com/llmware-ai/llmware/blob/main/examples/Getting_Started/configure_db.py). For vector embedding examples, you can use ChromaDB, LanceDB or FAISS - all of which provide no-install options - just start using. + + +- **Install DB dependencies**: we provide a number of Docker-Compose scripts which can be used, or follow install instructions provided by the database - generally easiest to install locally with Docker. + + +**LLMWare File Storage** + +- llmware stores a variety of artifacts during its operation locally in the /llmware_data path, which can be found as follows: + +```python +from llmware.configs import LLMWareConfig +llmware_fp = LLMWareConfig().get_llmware_path() +print("llmware_data path: ", llmware_fp) +``` + +- to change the llmware path, we can change both the 'home' path, which is the main filepath, and the 'llmware_data' path name +as follows: + +```python + +from llmware.configs import LLMWareConfig + +# changing the llmware home path - change home + llmware_path_name +LLMWareConfig().set_home("/my/new/local/home/path") +LLMWareConfig().set_llmware_path_name("llmware_data2") + +# check the new llmware home path +llmware_fp = LLMWareConfig().get_llmware_path() +print("updated llmware path: ", llmware_fp) + +``` + +___ + +___ +**Local Models** + +- LLMWare treats open source and locally deployed models as "first class citizens" with all classes, methods and examples designed to work first with smaller, specialized, locally-deployed models. +- By default, most models are pulled from public HuggingFace repositories, and cached locally. LLMWare will store all models locally at the /llmware_data/model_repo path, with all assets found in a folder tree with the models name. +- If a Pytorch model is pulled from HuggingFace, then it will appear in the default HuggingFace /.cache path. +- To view the local model path: + +```python +from llmware.configs import LLMWareConfig + +model_fp = LLMWareConfig().get_model_repo_path() +print("model repo path: ", model_fp) + +``` + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/getting_started/working_with_docker.md b/docs/getting_started/working_with_docker.md new file mode 100644 index 0000000..3cdb18f --- /dev/null +++ b/docs/getting_started/working_with_docker.md @@ -0,0 +1,157 @@ +--- +layout: default +title: Working with Docker +parent: Getting Started +nav_order: 6 +permalink: /getting_started/working_with_docker +--- + +# Working with Docker Scripts + +This section is a short guide on setting up a Linux environment with Docker and running LLMWare examples with different database systems. + +## 1. Python and Pip +Python should come installed with your Linux environment. + +To install Pip, run the following: +``` +sudo apt-get update +sudo apt-get -y install python3-pip +pip3 install --upgrade pip +``` + +## 2. Docker and Docker Compose +The latest versions of Docker and Docker Comopse should be installed to be able to use the Docker Compose files in the LLMWare repository. + +Instructions to install Docker: https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-on-ubuntu-20-04 (Steps 1-2) +Note: Step 1 is necessary, Step 2 is optional but we highly recommend it. + +Instructions to install Docker Compose: https://www.digitalocean.com/community/tutorials/how-to-install-and-use-docker-compose-on-ubuntu-20-04 (Step 1) +Note: replace the URL in the `curl` command with the latest download from https://github.com/docker/compose/releases. + +Check that Docker is running on your system: +``` +sudo systemctl status docker +``` + +## 3. Running Docker Compose files +`cd` into the repository and ensure that you can see files of the format `docker-compose-database-name.yaml`. + +To run a Compose file: +``` +docker-compose -f docker-compose-database-name.yaml up -d +``` + +Check that the container is running: +``` +docker ps +``` +Note: this will list only the the containers that are currently running, add the `-a` flag (`docker ps -a`) to list all containers (even those that are stopped). + +## 4. Test with Examples +The Compose files currently support 6 database systems: +- Mongo +- Postgres/PG Vector +- Neo4j +- Milvus +- Qdrant +- Redis + +Note: PG Vector is an alias for Postgres and is used for vector embeddings. + +1. Mongo and Postgres are used as the active database to store library text collections. +2. PG Vector, Neo4j, Milvus, Qdrant and Redis are used as the vector database to store vector embeddings. + +To test that the containers are working as intended, you can modify an example provided in the LLMWare repository. The simplest example to do this is `fast_start/example-2-build_embeddings.py`. + +Open the file in an editor. +1. Change the argument passed in as the active database on line 128 to an appropriate active database (Mongo or Postgres). +2. Change the argument passed in as the vector database on line 138 to an appropriate vector database (PG Vector, Neo4j, Milvus, Qdrant or Redis). + +Run the example with these changes, and you should see updates in the terminal indicating that the embeddings are being generated correctly. + +Note: It is possible that you will see an error: +``` +llmware.exceptions.EmbeddingModelNotFoundException: Embedding model for 'example2_library' could not be located +``` +In this case, use a unique name for the library name passed in on line 147. + +## 5. Stopping/Deleting Containers +To stop a container, run: +``` +docker stop container_ID_OR_container_name +``` + +To delete a container, run: +``` +docker rm container_ID_OR_container_name +``` + +Note: passing in either the ID or the name will work. + +To find the ID or name of a container, run: +``` +docker ps -a +``` + +--- + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..5228f65 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,163 @@ +--- +layout: default +title: Home | llmware +nav_order: 1 +description: llmware is an integrated framework with over 50+ models for quickly developing LLM-based applications including Retrieval Augmented Generation (RAG) and Multi-Step Orchestration of Agent Workflows. +permalink: / +--- +## Welcome to +
    +
  • + llmware +
    • +
+ +## 🧰🛠️🔩The Ultimate Toolkit for Enterprise RAG Pipelines with Small, Specialized Models + +From quickly building POCs to scalable LLM Apps for the enterprise, LLMWare is packed with all the tools you need. + +`llmware` is an integrated framework with over 50+ small, specialized, open source models for quickly developing LLM-based applications including Retrieval Augmented Generation (RAG) and Multi-Step Orchestration of Agent Workflows. + +This project provides a comprehensive set of tools that anyone can use - from a beginner to the most sophisticated AI developer - to rapidly build industrial-grade, knowledge-based enterprise LLM applications. + +Our specific focus is on making it easy to integrate open source small specialized models and connecting enterprise knowledge safely and securely. + + +## Getting Started + +1. Install llmware - `pip3 install llmware` + + +2. Make sure that you are running on a [supported platform](https://www.github.com/llmware-ai/llmware/tree/main/docs/getting_started/platforms.md#platform-support). + + +3. Learn by example: + + -- [Fast Start examples](https://www.github.com/llmware-ai/llmware/tree/main/fast_start) - structured set of 6 examples (with no DB installations required) to learn the main concepts of RAG with LLMWare - each example has extensive comments, and a supporting video on Youtube to walk you through it. + + -- [Getting Started examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started) - heavily-annotated examples that review many getting started elements - selecting a database, loading sample files, working with libraries, and how to use the Model Catalog. + + -- [Use Case examples](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases) - longer examples that integrate several components of LLMWare to provide a framework for a solution for common use case patterns. + + -- Dive into specific area of interest - [Parsing](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing) - [Models](https://www.github.com/llmware-ai/llmware/tree/main/examples/Models) - [Prompts](https://www.github.com/llmware-ai/llmware/tree/main/examples/Models) - [Agents](https://www.github.com/llmware-ai/llmware/tree/main/examples/SLIM-Agents) - and many more ... + + +4. We provide extensive [sample files](https://www.github.com/llmware-ai/tree/main/examples/Getting_Started/loading_sample_files.py) integrated into the examples, so you can copy-paste-run, and quickly validate that the installation is set up correctly, and to start seeing key classes and methods in action. We would encourage you to start with the 'out of the box' example first, and then use the example as the launching point for inserting your documents, models, queries, and workflows. + + +5. Learn by watching: check out the [LLMWare Youtube channel](https://www.youtube.com/@llmware). + + +6. Share with the community: join us on [Discord](https://discord.gg/MhZn5Nc39h). + + +[Install llmware](#install-llmware){: .btn .btn-primary .fs-5 .mb-4 .mb-md-0 .mr-2 } +[Common Setup & Configuration Items](#platform-support){: .btn .fs-5 .mb-4 .mb-md-0 } +[Architecture](architecture.md/#llmware-architecture){: .btn .fs-5 .mb-4 .mb-md-0 } +[View llmware on GitHub](https://www.github.com/llmware-ai/llmware/tree/main){: .btn .fs-5 .mb-4 .mb-md-0 } +[Open an Issue on GitHub](https://www.github.com/llmware-ai/llmware/issues){: .btn .fs-5 .mb-4 .mb-md-0 } + + + +# Install llmware + +___ +**Using Pip Install** + +- Installing llmware is easy: `pip3 install llmware` + + +- If you prefer, we also provide a set of recent wheels in the [wheel archives](https://www.github.com/llmware-ai/llmware/tree/main/wheel_archives) in this repository, which can be downloaded individually and used as follows: + +```bash +pip3 install llmware-0.2.12-py3-none-any.wheel +```` + +- We generally keep the main branch of this repository current with all changes, but we only publish new wheels to PyPi approximately once per week + +___ + +___ +**Cloning the Repository** + +- If you prefer to clone the repository: + +```bash +git clone git@github.com:llmware-ai/llmware.git +``` + +- The llmware package is contained entirely in the /llmware folder path, so you should be able to drop this folder (with all of its contents) into a project tree, and use the llmware module essentially the same as a pip install. + +- Please ensure that you are capturing and updating the /llmware/lib folder, which includes required compiled shared libraries. If you prefer, you can keep only those libs required for your OS platform. + +- After cloning the repo, we provide a short 'welcome to llmware' automation script, which can be used to install the projects requirements (from llmware/requirements.txt), install several optional dependencies that are commonly used in examples, copy several good 'getting started' examples into the root folder, and then run a 'welcome_example.py' script to get started using our models. To use the "welcome to llmware" script: + +Windows: +```bash +.\welcome_to_llmware_windows.sh +``` + +Mac/Linux: +```bash +sh ./welcome_to_llmware.sh +``` + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/advanced_techniques_for_rag.md b/docs/learn/advanced_techniques_for_rag.md new file mode 100644 index 0000000..983bf9a --- /dev/null +++ b/docs/learn/advanced_techniques_for_rag.md @@ -0,0 +1,85 @@ +--- +layout: default +title: Advanced RAG +parent: Learn +nav_order: 4 +description: overview of the major modules and classes of LLMWare +permalink: /learn/advanced_techniques_for_rag +--- +llmware Youtube Video Channel +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Advanced RAG Techniques ** +- [Best Small RAG Model - Bling-Phi-3](https://youtu.be/cViMonCAeSc?si=L6jX0sRdZAmKtRcz) +- [Agent Automation with Web Services for Financial Research](https://youtu.be/l0jzsg1_Ik0?si=oBGtALHLplouY9x2) +- [Voice Transcription and Automated Analysis of Greatest Speeches Dataset](https://youtu.be/5y0ez5ZBpPE?si=PAaCIqYou8nCGxYG) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part I](https://youtu.be/7oMTGhSKuNY?si=_KSjuBnqArvWzYbx) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part II- Code Experiments](https://youtu.be/iXp1tj-pPjM?si=3ZeMgipY0vJDHIMY) +- [Text2SQL Intro](https://youtu.be/BKZ6kO2XxNo?si=tXGt63pvrp_rOlIP) +- [Pop up LLMWare Inference Server](https://www.youtube.com/watch?v=qiEmLnSRDUA&t=20s) +- [Hardest Problem in RAG - handling 'Not Found'](https://youtu.be/slDeF7bYuv0?si=j1nkdwdGr5sgvUtK) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/core_rag_scenarios_running_locally.md b/docs/learn/core_rag_scenarios_running_locally.md new file mode 100644 index 0000000..250e18c --- /dev/null +++ b/docs/learn/core_rag_scenarios_running_locally.md @@ -0,0 +1,86 @@ +--- +layout: default +title: Core RAG Scenarios Running Locally +parent: Learn +nav_order: 2 +description: overview of the major modules and classes of LLMWare +permalink: /learn/core_rag_scenarios_running_locally +--- +Core RAG Scenarios Run Locally +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Core RAG Scenarios** + +- [Use small LLMs for RAG for Contract Analysis (feat. LLMWare)](https://www.youtube.com/watch?v=8aV5p3tErP0) +- [Invoice Processing with LLMware](https://www.youtube.com/watch?v=VHZSaBBG-Bo&t=10s) +- [Evaluate LLMs for RAG with LLMWare](https://www.youtube.com/watch?v=s0KWqYg5Buk&t=105s) +- [Fast Start to RAG with LLMWare Open Source Library](https://www.youtube.com/watch?v=0naqpH93eEU) +- [Use Retrieval Augmented Generation (RAG) without a Database](https://www.youtube.com/watch?v=tAGz6yR14lw) +- [Best Small RAG Model - Bling-Phi-3](https://youtu.be/cViMonCAeSc?si=L6jX0sRdZAmKtRcz) +- [RAG with BLING on your laptop](https://www.youtube.com/watch?v=JjgqOZ2v5oU) +- [DRAGON-7B-Models](https://www.youtube.com/watch?v=d_u7VaKu6Qk&t=37s) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/integrated_voice_transcription_with_whisper_cpp.md b/docs/learn/integrated_voice_transcription_with_whisper_cpp.md new file mode 100644 index 0000000..a681900 --- /dev/null +++ b/docs/learn/integrated_voice_transcription_with_whisper_cpp.md @@ -0,0 +1,78 @@ +--- +layout: default +title: Voice Transcription with Whisper CPP +parent: Learn +nav_order: 6 +description: overview of the major modules and classes of LLMWare +permalink: /learn/integrated_voice_transcription_with_whisper_cpp +--- +Integrated Voice Transcription with Whisper CPP +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Using Whisper CPP Models** +- [Getting Started with Whisper.CPP](https://youtu.be/YG5u5AOU9MQ?si=5xQYZCILPSiR8n4s) +- [Voice Transcription and Automated Analysis of Greatest Speeches Dataset](https://youtu.be/5y0ez5ZBpPE?si=PAaCIqYou8nCGxYG) + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/learn.md b/docs/learn/learn.md new file mode 100644 index 0000000..fe71b66 --- /dev/null +++ b/docs/learn/learn.md @@ -0,0 +1,118 @@ +--- +layout: default +title: Learn +nav_order: 4 +has_children: true +description: key learning resources +permalink: /learn +--- +Learn: Youtube Video Series +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Some of our most recent videos** +- [Best Small RAG Model - Bling-Phi-3](https://youtu.be/cViMonCAeSc?si=L6jX0sRdZAmKtRcz) +- [Agent Automation with Web Services for Financial Research](https://youtu.be/l0jzsg1_Ik0?si=oBGtALHLplouY9x2) +- [Voice Transcription and Automated Analysis of Greatest Speeches Dataset](https://youtu.be/5y0ez5ZBpPE?si=PAaCIqYou8nCGxYG) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part I](https://youtu.be/7oMTGhSKuNY?si=_KSjuBnqArvWzYbx) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part II- Code Experiments](https://youtu.be/iXp1tj-pPjM?si=3ZeMgipY0vJDHIMY) + +🎬 **Using Agents, Function Calls and SLIM models** +- [SLIMS Playlist](https://youtube.com/playlist?list=PL1-dn33KwsmAHWCWK6YjZrzicQ2yR6W8T&si=TSFGqQ3ObOO5vDde) +- [Agent-based Complex Research Analysis](https://youtu.be/y4WvwHqRR60?si=jX3KCrKcYkM95boe) +- [Getting Started with SLIMs (with code)](https://youtu.be/aWZFrTDmMPc?si=lmo98_quo_2Hrq0C) +- [SLIM Models Intro](https://www.youtube.com/watch?v=cQfdaTcmBpY) +- [Text2SQL Intro](https://youtu.be/BKZ6kO2XxNo?si=tXGt63pvrp_rOlIP) +- [Pop up LLMWare Inference Server](https://www.youtube.com/watch?v=qiEmLnSRDUA&t=20s) +- [Hardest Problem in RAG - handling 'Not Found'](https://youtu.be/slDeF7bYuv0?si=j1nkdwdGr5sgvUtK) +- [Extract Information from Earnings Releases](https://youtu.be/d6HFfyDk4YE?si=VmnIiWFmgBtR4DxS) +- [Summary Function Calls](https://youtu.be/yNg_KH5cPSk?si=Yl94tp_vKA8e7eT7) +- [Boolean Yes-No Function Calls](https://youtu.be/jZQZMMqAJXs?si=lU4YVI0H0tfc9k6e) +- [Autogenerate Topics, Tags and NER](https://youtu.be/N6oOxuyDsC4?si=vo2Fd8VG5xTbH4SD) + +🎬 **Using GGUF Models** +- [Using LM Studio Models](https://www.youtube.com/watch?v=h2FDjUyvsKE) +- [Using Ollama Models](https://www.youtube.com/watch?v=qITahpVDuV0) +- [Use any GGUF Model](https://www.youtube.com/watch?v=9wXJgld7Yow) +- [Background on GGUF Quantization & DRAGON Model Example](https://www.youtube.com/watch?v=ZJyQIZNJ45E) +- [Getting Started with Whisper.CPP](https://youtu.be/YG5u5AOU9MQ?si=5xQYZCILPSiR8n4s) + +🎬 **Core RAG Scenarios Running Locally** +- [RAG with BLING on your laptop](https://www.youtube.com/watch?v=JjgqOZ2v5oU) +- [DRAGON-7B-Models](https://www.youtube.com/watch?v=d_u7VaKu6Qk&t=37s) +- [Use small LLMs for RAG for Contract Analysis (feat. LLMWare)](https://www.youtube.com/watch?v=8aV5p3tErP0) +- [Invoice Processing with LLMware](https://www.youtube.com/watch?v=VHZSaBBG-Bo&t=10s) +- [Evaluate LLMs for RAG with LLMWare](https://www.youtube.com/watch?v=s0KWqYg5Buk&t=105s) +- [Fast Start to RAG with LLMWare Open Source Library](https://www.youtube.com/watch?v=0naqpH93eEU) +- [Use Retrieval Augmented Generation (RAG) without a Database](https://www.youtube.com/watch?v=tAGz6yR14lw) + + +🎬 **Parsing, Embedding, Data Pipelines and Extraction** +- [Ingest PDFs at Scale](https://www.youtube.com/watch?v=O0adUfrrxi8&t=10s) +- [Install and Compare Multiple Embeddings with Postgres and PGVector](https://www.youtube.com/watch?v=Bncvggy6m5Q) +- [Intro to Parsing and Text Chunking](https://youtu.be/2xDefZ4oBOM?si=YZzBUjDfQ0839EVF) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- + diff --git a/docs/learn/other_topics.md b/docs/learn/other_topics.md new file mode 100644 index 0000000..c8678b2 --- /dev/null +++ b/docs/learn/other_topics.md @@ -0,0 +1,85 @@ +--- +layout: default +title: Other Topics +parent: Learn +nav_order: 7 +description: overview of the major modules and classes of LLMWare +permalink: /learn/other_topics +--- +Other Notable Videos and Topics +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Some of our most recent videos** +- [Fast Local Chatbot with Phi-3-GGUF](https://youtu.be/gzzEVK8p3VM?si=HTMWQtN9XuaqjmpK) +- [Document Summarization](https://youtu.be/Ps3W-P9A1m8?si=mHvCcHvrKzndaNul) +- [Agent Server](https://youtu.be/nsA6-ZdnkXg?si=v7iGhC_rpj8TWbbl) +- [Best Small RAG Model - Bling-Phi-3](https://youtu.be/cViMonCAeSc?si=L6jX0sRdZAmKtRcz) +- [Agent Automation with Web Services for Financial Research](https://youtu.be/l0jzsg1_Ik0?si=oBGtALHLplouY9x2) +- [Voice Transcription and Automated Analysis of Greatest Speeches Dataset](https://youtu.be/5y0ez5ZBpPE?si=PAaCIqYou8nCGxYG) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part I](https://youtu.be/7oMTGhSKuNY?si=_KSjuBnqArvWzYbx) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part II- Code Experiments](https://youtu.be/iXp1tj-pPjM?si=3ZeMgipY0vJDHIMY) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/parsing_embedding_data_extraction.md b/docs/learn/parsing_embedding_data_extraction.md new file mode 100644 index 0000000..76fc2d0 --- /dev/null +++ b/docs/learn/parsing_embedding_data_extraction.md @@ -0,0 +1,82 @@ +--- +layout: default +title: Parsing Embedding and Data Extraction +parent: Learn +nav_order: 5 +description: overview of the major modules and classes of LLMWare +permalink: /learn/parsing_embedding_data_extraction +--- +Parsing, Embedding, and Data Extraction +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Parsing, Embedding, Data Pipelines and Extraction** +- [Advanced Parsing Techniques](https://youtu.be/dEsw8V_YBYY?si=B0GTVNhwfBYWkXyf) +- [Ingest PDFs at Scale](https://www.youtube.com/watch?v=O0adUfrrxi8&t=10s) +- [Install and Compare Multiple Embeddings with Postgres and PGVector](https://www.youtube.com/watch?v=Bncvggy6m5Q) +- [Intro to Parsing and Text Chunking](https://youtu.be/2xDefZ4oBOM?si=YZzBUjDfQ0839EVF) +- [Voice Transcription and Automated Analysis of Greatest Speeches Dataset](https://youtu.be/5y0ez5ZBpPE?si=PAaCIqYou8nCGxYG) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/using_agents_functions_slim_models.md b/docs/learn/using_agents_functions_slim_models.md new file mode 100644 index 0000000..bf7767f --- /dev/null +++ b/docs/learn/using_agents_functions_slim_models.md @@ -0,0 +1,90 @@ +--- +layout: default +title: Using Agents & Function Calls with SLIM Models +parent: Learn +nav_order: 1 +description: overview of the major modules and classes of LLMWare +permalink: /learn/using_agents_functions_slim_models +--- +Using Agents, Function Calls and SLIM Models +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Using Agents, Function Calls and SLIM models** +- [Agent Automation with Web Services for Financial Research](https://youtu.be/l0jzsg1_Ik0?si=oBGtALHLplouY9x2) +- [Sentiment Analysis](https://youtu.be/ERCHP21oAN8?si=fp6D4Tk9J2HdDRXa) +- [SLIMS Playlist](https://youtube.com/playlist?list=PL1-dn33KwsmAHWCWK6YjZrzicQ2yR6W8T&si=TSFGqQ3ObOO5vDde) +- [Agent-based Complex Research Analysis](https://youtu.be/y4WvwHqRR60?si=jX3KCrKcYkM95boe) +- [Getting Started with SLIMs (with code)](https://youtu.be/aWZFrTDmMPc?si=lmo98_quo_2Hrq0C) +- [SLIM Models Intro](https://www.youtube.com/watch?v=cQfdaTcmBpY) +- [Text2SQL Intro](https://youtu.be/BKZ6kO2XxNo?si=tXGt63pvrp_rOlIP) +- [Pop up LLMWare Inference Server](https://www.youtube.com/watch?v=qiEmLnSRDUA&t=20s) +- [Hardest Problem in RAG - handling 'Not Found'](https://youtu.be/slDeF7bYuv0?si=j1nkdwdGr5sgvUtK) +- [Extract Information from Earnings Releases](https://youtu.be/d6HFfyDk4YE?si=VmnIiWFmgBtR4DxS) +- [Summary Function Calls](https://youtu.be/yNg_KH5cPSk?si=Yl94tp_vKA8e7eT7) +- [Boolean Yes-No Function Calls](https://youtu.be/jZQZMMqAJXs?si=lU4YVI0H0tfc9k6e) +- [Autogenerate Topics, Tags and NER](https://youtu.be/N6oOxuyDsC4?si=vo2Fd8VG5xTbH4SD) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/docs/learn/using_quantized_gguf_models.md b/docs/learn/using_quantized_gguf_models.md new file mode 100644 index 0000000..8a635fe --- /dev/null +++ b/docs/learn/using_quantized_gguf_models.md @@ -0,0 +1,87 @@ +--- +layout: default +title: Using Quantized GGUF Models +parent: Learn +nav_order: 3 +description: overview of the major modules and classes of LLMWare +permalink: /learn/using_quantized_gguf_models +--- +Using Quantized GGUF Models +--- + +**Tutorial Videos** - check out our Youtube channel for high-impact 5-10 minute tutorials on the latest examples. + +Check back often as this list is always growing ... + +🎬 **Using GGUF Models** +- [Using LM Studio Models](https://www.youtube.com/watch?v=h2FDjUyvsKE) +- [Using Ollama Models](https://www.youtube.com/watch?v=qITahpVDuV0) +- [Use any GGUF Model](https://www.youtube.com/watch?v=9wXJgld7Yow) +- [Background on GGUF Quantization & DRAGON Model Example](https://www.youtube.com/watch?v=ZJyQIZNJ45E) +- [Getting Started with Whisper.CPP](https://youtu.be/YG5u5AOU9MQ?si=5xQYZCILPSiR8n4s) +- [Best Small RAG Model - Bling-Phi-3](https://youtu.be/cViMonCAeSc?si=L6jX0sRdZAmKtRcz) +- [Agent Automation with Web Services for Financial Research](https://youtu.be/l0jzsg1_Ik0?si=oBGtALHLplouY9x2) +- [Voice Transcription and Automated Analysis of Greatest Speeches Dataset](https://youtu.be/5y0ez5ZBpPE?si=PAaCIqYou8nCGxYG) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part I](https://youtu.be/7oMTGhSKuNY?si=_KSjuBnqArvWzYbx) +- [Are you prompting wrong for RAG - Stochastic Sampling-Part II- Code Experiments](https://youtu.be/iXp1tj-pPjM?si=3ZeMgipY0vJDHIMY) + + +# More information about the project - [see main repository](https://www.github.com/llmware-ai/llmware.git) + + +# About the project + +`llmware` is © 2023-{{ "now" | date: "%Y" }} by [AI Bloks](https://www.aibloks.com/home). + +## Contributing +Please first discuss any change you want to make publicly, for example on GitHub via raising an [issue](https://github.com/llmware-ai/llmware/issues) or starting a [new discussion](https://github.com/llmware-ai/llmware/discussions). +You can also write an email or start a discussion on our Discrod channel. +Read more about becoming a contributor in the [GitHub repo](https://github.com/llmware-ai/llmware/blob/main/CONTRIBUTING.md). + +## Code of conduct +We welcome everyone into the ``llmware`` community. +[View our Code of Conduct](https://github.com/llmware-ai/llmware/blob/main/CODE_OF_CONDUCT.md) in our GitHub repository. + +## ``llmware`` and [AI Bloks](https://www.aibloks.com/home) +``llmware`` is an open source project from [AI Bloks](https://www.aibloks.com/home) - the company behind ``llmware``. +The company offers a Software as a Service (SaaS) Retrieval Augmented Generation (RAG) service. +[AI Bloks](https://www.aibloks.com/home) was founded by [Namee Oberst](https://www.linkedin.com/in/nameeoberst/) and [Darren Oberst](https://www.linkedin.com/in/darren-oberst-34a4b54/) in October 2022. + +## License + +`llmware` is distributed by an [Apache-2.0 license](https://www.github.com/llmware-ai/llmware/blob/main/LICENSE). + +## Thank you to the contributors of ``llmware``! +
    +{% for contributor in site.github.contributors %} +
  • + + {{ contributor.login }} + +
    • +{% endfor %} +
+ + +--- +
    +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
  • + +
    • +
+--- diff --git a/llmware/__init__.py b/llmware/__init__.py new file mode 100644 index 0000000..e4e4b16 --- /dev/null +++ b/llmware/__init__.py @@ -0,0 +1,26 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The llmware package provides tools to build large language model (LLM) applications, this includes +a custom set of LLMs that are small and open source. + +The llmware package aspires to be a middleware in LLM applications. In other words, it provides the +infrastructure between the components, such as the models, the prompts, the text databases, and +the vector databases. +""" + + +__version__ = '0.4.6' +__author__ = 'llmware' +__license__ = 'Apache 2.0 License' diff --git a/llmware/agents.py b/llmware/agents.py new file mode 100755 index 0000000..b85a4b9 --- /dev/null +++ b/llmware/agents.py @@ -0,0 +1,1676 @@ + +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The agents module implements the two classes LLMfx and SQLTables, where LLMfx manages +Structured Language Instruction Models (SLIMs), the agents and SQLTables handles +creating and accessing external SQL data. LLmfx currently only supports SLIM models, other model +classes will be added over time. And SQLTables is an experimental feature for creating and accessing SQLite. + +A Structured Language Instruction Model, SLIM for short, is a small specialized multi-modal LLM for function +calling and multi-step workflows. +""" + +import shutil +import logging +import gc +import re +import csv +import os +import sqlite3 +import json + +from llmware.models import ModelCatalog, _ModelRegistry +from llmware.util import CorpTokenizer, AgentWriter +from llmware.resources import CustomTable +from llmware.configs import LLMWareConfig, SQLiteConfig, ModelNotFoundException + +logger = logging.getLogger(__name__) +logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__)) + + +class LLMfx: + + """ + + ``LLMfx`` provides a high-level orchestration abstraction that implements multi-model, multi-step processes + with the ability to load and orchestrate multiple SLIM models as tools with centralized journaling, + structured work management and information aggregation. Currently, LLMfx only supports SLIM classifier + models, support for additional model classes will be added over time. + + Parameters + ---------- + api_key : str, optional, default=None + Sets the API key that used by the ``ModelCatalog`` to load models and logs. + + verbose : bool, optional, default=True + Sets whether ``agent_writer.write`` statements should be executed or not, e.g. if ```verbose=True```, then new + events that are written to the journal are written to stdout. + + analyze_mode : bool, optional, default=True + Sets whether logits should be retrieved when a tool is called with ``exec_function_call``. + + Returns + ------- + llmfx : LLMfx + A new ``LLMfx`` object. + + """ + + def __init__(self, api_key=None, verbose=True, analyze_mode=True): + + self.agent_writer = AgentWriter() + + if self.agent_writer.mode == "file": + logger.info(f"update: AgentWriter mode set to file - writing agent work process to: " + f"{os.path.join(self.agent_writer.fp_base, self.agent_writer.fn)}" + f"\nTo change file: `LLMWareConfig().set_agent_file('new_file_name.txt')`" + f"\nTo change to screen: `LLMWareConfig().set_agent_log('screen')") + + if verbose: + self.agent_writer.write("update: Launching LLMfx process") + + self._supported_tools = _ModelRegistry().get_llm_fx_tools_list() + self._default_tool_map = _ModelRegistry().get_llm_fx_mapping() + + for tools in self._supported_tools: + setattr(self, tools + "_model", None) + + self.work_queue = [] + self.work_iteration = 0 + + self.verbose = verbose + self.analyze_mode = analyze_mode + + # report is a list of dictionaries, with each dictionary linked to a work item number + # reports are automatically aggregated through the lifecycle of the object + self.report = [] + + # response list provides a list of the llm tool responses + self.response_list = [] + + # research list provides a list of any research gathered (specifically from SQLTables currently) + self.research_list = [] + + # journal keeps a running journal output used in 'verbose' mode to the screen display + self.journal = [] + self.step = 0 + + journal_update = f"creating object - ready to start processing." + self.write_to_journal(journal_update) + + self.tools_deployed = [] + self.inference_calls = 0 + + # set by default to localhost, 8080 and using 'demo-test' api_key + self.api_endpoint = "http://127.0.0.1/8080" + self.api_key = api_key + self.api_exec = False + + self.sql_query = None + + # check for llmware path & create if not already set up, e.g., "first time use" + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + logger.info("Agent - Setting up LLMWare Workspace.") + + def register_api_endpoint(self, api_endpoint = None, api_key=None, endpoint_on=True): + + self.api_endpoint = api_endpoint + self.api_key=api_key + self.api_exec = endpoint_on + return True + + def switch_endpoint_on(self): + self.api_exec = True + return True + + def switch_endpoint_off(self): + self.api_exec = False + return True + + def update_tool_map(self, tool_type, tool_name): + + """ Updates tool mapping for LLMfx instance - enables swapping in other models. """ + + if tool_type: + if tool_type in self._supported_tools: + + # unload tool if currently being used + self.unload_tool(tool_type) + + # create new mapping + self._default_tool_map.update({tool_type: tool_name}) + + # load new tool + self.load_tool(tool_type) + + return self + + def clear_work(self): + + """ Detaches any loaded text work and resets the iteration number. """ + + self.work_queue = [] + self.work_iteration = 0 + + journal_update = f"clearing work queue - reset" + self.write_to_journal(journal_update) + + return True + + def set_work_iteration(self, num): + + """ Sets the work iteration number. """ + + if num < len(self.work_queue): + self.work_iteration = num + + journal_update = f"setting work iteration to entry - {str(num)}" + self.write_to_journal(journal_update) + + return True + + def top_of_work_queue(self): + + """ Sets the work iteration number to the last item in the work queue and returns this value. """ + + self.work_iteration = len(self.work_queue) - 1 + return self.work_iteration + + def increment_work_iteration(self): + + """ Increments the work iteration - will return None if nothing left in the processing queue. """ + + if (self.work_iteration + 1) < len(self.work_queue): + self.work_iteration += 1 + output_value = self.work_iteration + journal_update = f"incrementing work iteration to entry - {str(self.work_iteration)}" + else: + journal_update = f"completed all work processing" + output_value = None + + self.write_to_journal(journal_update) + + return output_value + + def _expand_report(self): + + """ Creates an incremental empty report dictionary in line with creation of a new work item. """ + + self.report.append({}) + return len(self.report) + + def load_work(self, text, text_key="text"): + + """ Flexible intake method accepts multiple forms of input text: + --if string, then packages as a dictionary, and adds to the work_queue + --if dictionary, then checks the keys and adds to the work_queue + --if list, then unpacks and iterates, adding each entry as a dictionary onto the work queue """ + + new_entries_created = 0 + + if isinstance(text, str): + new_entry = {"text": text, "file_source": "NA", "page_num": "NA"} + self.work_queue.append(new_entry) + new_entries_created += 1 + self._expand_report() + + if isinstance(text, dict): + if text_key in text and "file_source" in text and "page_num" in text: + self.work_queue.append(text) + new_entries_created += 1 + self._expand_report() + else: + if text_key not in text: + logging.warning("could not identify dictionary type.") + return -1 + else: + if "file_source" not in text: + text.update({"file_source": "NA"}) + if "page_num" not in text: + text.update({"page_num": "NA"}) + self.work_queue.append(text) + new_entries_created += 1 + self._expand_report() + + if isinstance(text, list): + # need to check the type of the entries in the list + for i, elements in enumerate(text): + + if isinstance(elements, str): + new_entry = {"text": elements, "file_source": "NA", "page_num": "NA"} + self.work_queue.append(new_entry) + new_entries_created += 1 + self._expand_report() + + if isinstance(elements, dict): + if text_key in elements and "file_source" in elements and "page_num" in elements: + self.work_queue.append(elements) + new_entries_created += 1 + self._expand_report() + else: + if text_key not in elements: + logging.warning("update: load - skipping - could not identify " + "dictionary type - %s", elements) + else: + if "file_source" not in elements: + elements.update({"file_source": "NA"}) + if "page_num" not in elements: + elements.update({"page_num": "NA"}) + self.work_queue.append(elements) + new_entries_created += 1 + self._expand_report() + + journal_update = f"loading new processing text - {str(new_entries_created)} new entries" + self.write_to_journal(journal_update) + + return self.work_queue + + def clear_state(self): + + """ Resets key state variables of LLMfx instance """ + + self.journal = [] + self.tools_deployed = [] + self.inference_calls = 0 + self.response_list = [] + # self.report = {} + self.report = [] + self.step = 0 + + return self + + def activity_summary(self): + + """ Provides an activity summary and writes to journal. """ + + activity_summary = {"inference_count": self.inference_calls, "tools_used": len(self.tools_deployed), + "tools": self.tools_deployed} + + journal_update = f"generating activity_summary - {str(activity_summary)}" + self.write_to_journal(journal_update) + + return activity_summary + + def show_report(self, iteration_num=None,add_source=True): + + """ Shows the gathered report so far, and writes to journal. """ + + output_report = [] + + if iteration_num: + + if not isinstance(iteration_num,list): + iteration_num = [iteration_num] + + # show specific report(s) + journal_update = f"showing selected reports - {str(iteration_num)}\n" + for n in iteration_num: + journal_update += f"showing gathered report - {str(self.report[n])}\n" + for key, value in self.report[n].items(): + journal_update += f"\t\t\t\t -- {key.ljust(20)} - {str(value).ljust(40)}\n" + + source_info = "" + if "file_source" in self.work_queue[n]: + source_info += self.work_queue[n]["file_source"] + if "page_num" in self.work_queue[n]: + source_info += " - page: " + str(self.work_queue[n]["page_num"]) + + key= "source_info" + value = source_info + if source_info: + journal_update += f"\t\t\t\t -- {key.ljust(20)} - {str(value).ljust(40)}\n" + + base_report = self.report[n] + if add_source: + base_report.update({"source": self.work_queue[n]}) + + output_report.append(base_report) + + self.write_to_journal(journal_update) + + else: + # show all reports + output_report = [] + journal_update = f"showing all gathered reports - {str(self.report)}\n" + for i, entries in enumerate(self.report): + journal_update += f"report - {str(i)} - {str(self.report[i])}\n" + for key, value in self.report[i].items(): + journal_update += f"\t\t\t\t -- {key.ljust(20)} - {str(value).ljust(40)}\n" + if add_source: + entries.update({"source": self.work_queue[i]}) + output_report.append(entries) + self.write_to_journal(journal_update) + # output_report = self.report + + return output_report + + def lookup_response_by_tool(self, tool_type): + + """ Looks up an item in the response list by tool type. """ + + output = [] + + for i, response in enumerate(self.response_list): + if response["tool"] == tool_type: + output.append(response) + + return output + + def follow_up_list(self, key=None, value=None): + + """ Analyzes response list and returns sub-set with matching 'key' and 'value' """ + + follow_up_list = [] + + if not key: + journal_update = f"building follow-up_list - looking for distinct work items\n" + else: + journal_update = f"building follow_up_list - looking for {key} - {value}\n" + + key_value_str = f"{key} - {value}" + + for i, response in enumerate(self.response_list): + if "llm_response" in response: + + work_num = response["work_iteration"] + text = response["text"] + + if key: + if key in response["llm_response"]: + if value in response["llm_response"][key]: + follow_up_list.append(work_num) + + journal_update += f"\t\t\t\t -- {key_value_str.ljust(20)} - {str(work_num)} - {str(text)}\n" + + else: + if work_num not in follow_up_list: + follow_up_list.append(work_num) + placeholder = "distinct_work_item" + journal_update += f"\t\t\t\t -- {placeholder.ljust(20)} - {str(work_num)} - {str(text)}\n" + + self.write_to_journal(journal_update) + + return follow_up_list + + def analyze_responses(self, key,value): + + """ Analyzes response list and returns sub-set with matching 'key' and 'value' """ + + journal_update = f"analyzing responses - looking for {key} - {value}\n" + + output_list = [] + key_value_str = f"{key} - {value}" + + for i,response in enumerate(self.response_list): + if "llm_response" in response: + if key in response["llm_response"]: + if value in response["llm_response"][key]: + output_list.append(response) + + cl = response["confidence_score"] + text = response["work_item"]["text"] + step = response["step"] + + journal_update += f"\t\t\t\t -- {key_value_str.ljust(20)} - {str(step)} - {str(text)}\n" + + self.write_to_journal(journal_update) + + return output_list + + def load_tool(self, tool_type, + # new options added + use_gpu=True, sample=False, get_logits=True, + max_output=100, temperature=0.0): + + """ Loads a single tool """ + + model = None + + if not self.api_exec: + + if tool_type in self._supported_tools: + + journal_update = f"loading tool - {tool_type}" + self.write_to_journal(journal_update) + + setattr(self, tool_type + "_model", + ModelCatalog().load_model(self._default_tool_map[tool_type],api_key=self.api_key, + sample=sample,use_gpu=use_gpu,get_logits=get_logits,max_output=max_output, + temperature=temperature)) + + model = getattr(self, tool_type + "_model") + + if tool_type not in self.tools_deployed: + self.tools_deployed.append(tool_type) + + else: + journal_update = f"api_exec mode = 'ON' - skipping - local loading of tool - {tool_type}" + self.write_to_journal(journal_update) + + return model + + def load_tool_list(self, tool_list): + + """ Loads a list of tool, typically at the start of a multi-step process. """ + + if not self.api_exec: + + for tool_type in tool_list: + + if tool_type in self._supported_tools: + + model = getattr(self, tool_type + "_model") + + if not model: + self.load_tool(tool_type) + else: + journal_update = f"api_exec mode = 'ON' - skipping - local loading of tool list - {str(tool_list)}" + self.write_to_journal(journal_update) + + return self + + def unload_tool(self, tool_type): + + """ Unloads a tool, which removes it from memory - useful in long-running processes + to be able to load and unload different tools. """ + + if not self.api_exec: + + if tool_type in self._supported_tools: + + journal_update = f"unloading tool - {tool_type}" + self.write_to_journal(journal_update) + + model = getattr(self, tool_type + "_model") + + if model: + + model.unload_model() + + delattr(self, tool_type + "_model") + setattr(self, tool_type + "_model", None) + gc.collect() + + else: + journal_update = f"api_exec mode = 'ON' - skipping - local 'unload' of model- {tool_type}" + self.write_to_journal(journal_update) + + return 0 + + def write_to_journal(self, journal_update): + + """ Adds an event to the running journal list and displays if in verbose mode. """ + + self.journal.append(journal_update) + self.step += 1 + + if self.verbose: + self.agent_writer.write(f"step - \t{str(self.step)} - \t{journal_update}") + + return True + + def exec_function_call(self, tool_type, text=None, function="classify", params=None, get_logits=True): + + """ Executes a function call on the selected tool type. """ + + value_output = {} + + if tool_type in self._supported_tools: + + journal_update = f"executing function call - deploying - {tool_type} " + self.write_to_journal(journal_update) + + if text: + # if text passed directly, then add to work queue + self.load_work(text) + # set work iteration to be the last item + self.top_of_work_queue() + + # pull from the work queue + work_dict = self.work_queue[self.work_iteration] + work_iter = self.work_iteration + text = work_dict["text"] + + if not self.analyze_mode: + get_logits = False + + if not self.api_exec: + + model = getattr(self, tool_type + "_model") + + # if model not yet loaded, then load in-line + if not model: + model = self.load_tool(tool_type) + + function_call = getattr(model, "function_call") + + response = function_call(text, function=function, params=params, get_logits=get_logits) + + else: + + # send to api agent server + response = self.fx_over_api_endpoint(context=text,tool_type=tool_type, function=function,params=params, + get_logits=get_logits) + + self.inference_calls += 1 + output_response = {} + logit_analysis = {} + + if response: + + if "llm_response" in response: + + llm_response = response["llm_response"] + output_type = response["usage"]["type"] + usage= response["usage"] + + if response["usage"]["type"] == "dict": + dict_output = True + self.report[work_iter] = self.report[work_iter] | response["llm_response"] + + elif response["usage"]["type"] == "list" and tool_type == "summary": + dict_output = True + self.report[work_iter] = self.report[work_iter] | {"summary": response["llm_response"]} + + else: + logging.warning("update: could not automatically convert to dictionary - " + "keeping as string output") + dict_output = False + + # assemble output + value_output.update({"llm_response": llm_response,"dict_output": dict_output}) + + # start journaling update + journal_update = f"executing function call - " \ + f"getting response - {tool_type}\n" + journal_update += f"\t\t\t\t -- llm_response - {str(llm_response)}\n" + journal_update += f"\t\t\t\t -- output type - {output_type}\n" + journal_update += f"\t\t\t\t -- usage - {usage}" + + self.write_to_journal(journal_update) + # end journaling + + # default - if not found/applied + confidence_score = -1 + + # load the model card + model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type] + model_card = ModelCatalog().lookup_model_card(model_name) + hf_tokenizer_name = model_card["tokenizer"] + + if get_logits: + logit_analysis = ModelCatalog().logit_analysis(response, model_card, + hf_tokenizer_name, + api_key=self.api_key) + + confidence_score = logit_analysis["confidence_score"] + ryg = logit_analysis["ryg_string"] + choices = logit_analysis["choices"] + + # will display and add to journal only the 'first' token choice + # choices for each token captured in 'logit_analysis' metadata + if len(choices) > 1: + choices = choices[0] + + marker_tokens = logit_analysis["marker_tokens"] + output_response.update({"logit_analysis": logit_analysis}) + + # start journaling update + journal_update = f"analyzing response - {tool_type}\n" + journal_update += f"\t\t\t\t -- confidence score - {str(confidence_score)}\n" + journal_update += f"\t\t\t\t -- analyzing response - {ryg}\n" + journal_update += f"\t\t\t\t -- analyzing response - {choices}" + if marker_tokens: + journal_update += "\n" + journal_update += f"\t\t\t\t -- analyzing response - {str(marker_tokens)}" + + self.write_to_journal(journal_update) + + value_output.update({"confidence_score": confidence_score}) + if marker_tokens: + value_output.update({"choices": marker_tokens}) + + # assemble output response dictionary + + output_response = {"step": self.step, "tool": tool_type, "inference": self.inference_calls, + "llm_response": llm_response} + + if get_logits: + output_response.update({"confidence_score": confidence_score}) + + output_response.update({"llm_usage": usage, "work_iteration": work_iter, "dict_output": dict_output}) + + for keys, values in work_dict.items(): + output_response.update({keys:values}) + + if get_logits: + output_response.update({"logit_analysis": logit_analysis}) + + # save to response list state tracker + self.response_list.append(output_response) + + else: + raise ModelNotFoundException(tool_type) + + return value_output + + def exec_multitool_function_call(self, tool_type_list, text=None, function="classify", params=None, + get_logits=True): + + """ Executes multiple function calls on the same text with a list of tools in tool_type_list """ + + output_list = [] + + for tool_type in tool_type_list: + + response = self.exec_function_call(tool_type,text=text,get_logits=get_logits, + params=params, function=function) + + output_list.append(response) + + return output_list + + def sentiment(self, text=None, params=None): + + """ Executes sentiment analysis on text, if passed directly, or will pull current work item from the + queue. Returns value output dictionary with sentiment classification, confidence score and choices. """ + + if not params: + # default parameter key + params = ["sentiment"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("sentiment", text=text, params=params) + + def topics(self, text=None, params=None): + + """ Executes topics analysis on text, if passed directly, or will pull current work item from the queue. + Returns value output dictionary with topics classification and confidence score. """ + + if not params: + # default parameter key + params = ["topic"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("topics", text=text, params=params) + + def named_entity_extraction(self, text=None, params=None): + + """ Executes named entity classification analysis on a text, if passed directly, or will pull current + work item from the queue. Returns value output dictionary with named entity classification and + confidence score. """ + + if not params: + # default parameter key + params = ["people", "place", "company", "misc"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("ner", text=text, params=params) + + def ner(self, text=None, params=None): + + """ Executes named entity classification analysis on a text, if passed directly, or will pull current + work item from the queue. Returns value output dictionary with named entity classification and + confidence score. """ + + #TODO: identical to "named_entity_extraction" method - should remove one of them + + if not params: + # default parameter key + params = ["people", "place", "company", "misc"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("ner", text=text, params=params) + + def ratings(self, text=None, params=None): + + """ Executes ratings classification analysis on a text of 1-5, if passed directly, or will pull current + work item from the queue. Returns value output dictionary with rating classification and + confidence score. """ + + if not params: + # default parameter key + params = ["rating"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("ratings", text=text, params=params) + + def emotions(self, text=None, params=None): + + """ Executes emotions classification analysis on a text, if passed directly, or will pull current + work item from the queue. Returns value output dictionary with emotions classification and + confidence score. """ + + if not params: + # default parameter key + params = ["emotions"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("emotions", text=text, params=params) + + def intent(self, text=None, params=None): + + """ Executes intent classification analysis on a text, if passed directly, or will pull current + work item from the queue. Returns value output dictionary with intent classification and + confidence score. """ + + if not params: + # default parameter key + params = ["intent"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("intent", text=text, params=params) + + def tags(self, text=None, params=None): + + """ Generates a list of relevant 'tag' information data points from a text, if passed directly, or + will pull current work item from the queue. Returns value output dictionary with list of key + highlighted points. """ + + if not params: + # default parameter key + params = ["tags"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("tags", text=text, params=params) + + def category(self, text=None, params=None): + + """ Generates a list of relevant business category information data points from a text, if passed + directly, or will pull current work item from the queue. Returns value output dictionary with list of + business category classification (usually a single entry, but possible for multiple entries). """ + + if not params: + # default parameter key + params = ["category"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("category", text=text, params=params) + + def sa_ner(self, text=None, params=None): + + """ Generates a dictionary with keys corresponding to 'sentiment' and 'named entity recognition' (NER) + identifiers in the next, such as people, organization, and place. """ + + if not params: + # default parameter key + params = ["sentiment, people, organization, place"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("sa-ner", text=text, params=params) + + def extract(self, text=None, params=None): + + """ Extract receives an input of a text passage and a custom parameter key, and generates a dictionary with + key corresponding to the 'custom parameter' key and a list of values associated with that key, extracted from + the text passage. """ + + if not params: + # default parameter key + params = ["key points"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("extract", text=text, params=params) + + def xsum(self, text=None, params=None): + + """ XSum or 'extreme summarization' receives an input text passage, and returns a dictionary with a 'xsum' + key and a value of a list with one string element, with the string element consisting of a short phrase, + title, headline that provides a concise summary of the text passage. """ + + if not params: + # default parameter key + params = ["xsum"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("xsum", text=text, params=params) + + def summarize(self, text=None, params=None): + + """ Summarizes receives an input text passage, and optional parameters to guide the summarization, and + returns a list of summary points from the text. """ + + if not params: + # default parameter key + params = ["key points (3)"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("summary", text=text, params=params) + + def boolean(self, text=None, params=None): + + """ Boolean receives an input text passage, a yes/no question as its parameter, and then returns a + dictionary with two keys - 'answer' and 'explain' with the 'answer' providing a yes/no classification, and the + explanation providing text from the passage that was used as the basis for the classification. + + Example: + text = "The stock was down sharply after the company announced an earnings miss." + params = "Is the stock down?" + + response = boolean(text=text, params=params) + + By default, the method will append the "explain" flag and include in the params to pass to the model + + """ + + if not params: + params = ["Is this true? (explain)"] + + if isinstance(params, str): + params = params + " (explain)" + params = [params] + + return self.exec_function_call("boolean", text=text, params=params) + + def nli(self, text1, text2, params=None): + + """ Executes a natural language inference classification on a text, if passed directly, or will pull current + work item from the queue. Returns value output dictionary with the NLI classification and + confidence score. """ + + if not params: + # default parameter key + params = ["evidence"] + + if isinstance(params, str): + params = [params] + + context = "Evidence: " + text1 + "\n" + "Conclusion: " + text2 + + return self.exec_function_call("nli", text=context, params=params) + + def q_gen(self, text=None, params=None): + + """ Executes a question-gen function call on a text, if passed directly, or will pull current work item from + the queue. Returns value output dictionary with the generated question. """ + + if not params: + params = ["question"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("q_gen", text=text, params=params) + + def qa_gen(self, text=None, params=None): + + """ Executes a question-answer gen function call on a text, if passed directly, or will pull current work + item from the queue. Returns value output dictionary with two keys - "question" and "answer" generated. """ + + if not params: + # default parameter key + params = ["question, answer"] + + if isinstance(params, str): + params = [params] + + return self.exec_function_call("qa_gen", text=text, params=params) + + def verify_llm_response(self, input_context, llm_response): + + """ Utility function to apply NLI to compare llm_response with the input context. """ + + return self.nli(input_context, llm_response) + + def answer(self, question, context=None, key=None): + + """ Executes an inference """ + + journal_update = f"executing function call - deploying - question-answer tool " + self.write_to_journal(journal_update) + + if context: + self.load_work(context) + + work_dict = self.work_queue[self.work_iteration] + text = work_dict["text"] + work_iter = self.work_iteration + + if not self.api_exec: + + model = getattr(self, "answer" + "_model") + + # insert change - load model in-line + # if model not yet loaded, then load in-line + if not model: + model = self.load_tool("answer") + # end - insert change + + inference = getattr(model, "inference") + + response = inference(question, add_context=text, add_prompt_engineering=True) + + else: + # route answer request over API + response = self.fx_over_api_endpoint(tool_type="answer", context=text, prompt=question) + + llm_response = re.sub("[\n\r]", "\t", response["llm_response"]) + + if not key: + self.report[work_iter].update({"answer": [llm_response]}) + answer_key = "answer" + else: + self.report[work_iter].update({key:[llm_response]}) + answer_key = key + + usage = response["usage"] + + self.inference_calls += 1 + + # start journaling update + journal_update = f"executing function call - " \ + f"getting response - question - {answer_key}\n" + journal_update += f"\t\t\t\t -- llm_response - {str(llm_response)}\n" + journal_update += f"\t\t\t\t -- output type - text\n" + journal_update += f"\t\t\t\t -- usage - {usage}" + + self.write_to_journal(journal_update) + + # assemble output response dictionary + + output_response = {"step": self.step, "tool": "answer", "inference": self.inference_calls, + "llm_response": llm_response} + + get_logits=False + + if get_logits: + confidence_score =-1 + output_response.update({"confidence_score": confidence_score}) + + output_response.update({"llm_usage": usage, "work_iteration": work_iter, "dict_output": False}) + + for keys, values in work_dict.items(): + output_response.update({keys:values}) + + if get_logits: + logit_analysis= {} + output_response.update({"logit_analysis": logit_analysis}) + + # save to response list state tracker + self.response_list.append(output_response) + + return output_response + + def sql(self, query, table_schema): + + """ Executes Text2Sql tool to convert query into SQL """ + + if table_schema: + self.load_work(table_schema) + self.top_of_work_queue() + + work_dict = self.work_queue[self.work_iteration] + table_schema = work_dict["text"] + work_iter = self.work_iteration + + # initial journal update + journal_update = f"executing function call - deploying - text-to-sql\n" + journal_update += f"\t\t\t\t -- query - {query}\n" + journal_update += f"\t\t\t\t -- table_schema - {table_schema}" + self.write_to_journal(journal_update) + + if not self.api_exec: + + model = getattr(self, "sql" + "_model") + + # insert change - load model in-line + # if model not yet loaded, then load in-line + if not model: + model = self.load_tool("sql") + # end - insert change + + inference = getattr(model, "inference") + + response = inference(query, add_context=table_schema, add_prompt_engineering=True) + + else: + response = self.fx_over_api_endpoint(tool_type="sql", context=table_schema, prompt=query) + + self.inference_calls += 1 + + llm_response = response["llm_response"] + + # quick clean up response to replace any potential error-generating double-quotes and replace with + # correct sql syntax for single-quotes + llm_response = re.sub('"', "'", llm_response) + + self.report[work_iter].update({"sql": [llm_response]}) + + usage = response["usage"] + + self.inference_calls += 1 + + # start journaling update + journal_update = f"executing function call - getting response - sql\n" + journal_update += f"\t\t\t\t -- llm_response - {str(llm_response)}\n" + journal_update += f"\t\t\t\t -- output type - text\n" + journal_update += f"\t\t\t\t -- usage - {usage}" + + self.write_to_journal(journal_update) + # end journaling + + # assemble output response dictionary + + output_response = {"step": self.step, "tool": "sql", "inference": self.inference_calls, + "llm_response": llm_response} + + # logits not yet activated for inference calls - TBD - set 'get_logits = False" for now + get_logits=False + if get_logits: + confidence_score =-1 + output_response.update({"confidence_score": confidence_score}) + + output_response.update({"llm_usage": usage, "work_iteration": work_iter, "dict_output": False}) + + for keys, values in work_dict.items(): + output_response.update({keys:values}) + + if get_logits: + logit_analysis= {} + output_response.update({"logit_analysis": logit_analysis}) + + # save to response list state tracker + self.response_list.append(output_response) + + return output_response + + def sql_checker(self, sql_query, custom_sql_checker=None): + + """ Implements a basic post processing check on text-2-sql generation to confirm that + the query is a SELECT statement and not a form of DB WRITE command. + + By passing a custom_sql_checker function, you can enhance this basic check. + + The custom_sql_checker function should accept a string sql_query as input, + and return two outputs: + + 1- confirmation: a boolean truth value of True/False to indicate whether to move ahead + 2- sql_query_updated: a return string that may be identical/modification of original sql query + + """ + + # if no red-flags identified, then will return True and original sql_query + confirmation = True + sql_query_updated = sql_query + + logger.debug(f"LLMfx - sql_checker - {sql_query} - being reviewed.") + + if custom_sql_checker: + confirmation, sql_query_updated = custom_sql_checker(sql_query) + + else: + + # reviews any SQL statement that does not start with SELECT + + if not sql_query.startswith("SELECT"): + + sql_tokens = sql_query.split(" ") + + logger.warning(f"LLMfx - sql_checker - sql query statement does not start " + f"with SELECT statement - {sql_query}") + + # this list can be enhanced + basic_write_commands = ["DROP", "INSERT", "CREATE", "DELETE", "ALTER"] + + for toks in sql_tokens: + + if toks.upper() in basic_write_commands: + logger.warning(f"LLMfx - sql_checker - sql query statement appears to create " + f"WRITE elements - {toks} - stopping.") + + confirmation = False + break + + return confirmation, sql_query_updated + + def query_custom_table(self, query, db=None,table=None,table_schema=None,db_name="llmware", + custom_sql_checker=None): + + """ Executes a text-to-sql query on a CustomTable database table. """ + + custom_table = CustomTable(db=db,table_name=table) + + if not table_schema: + if table: + table_schema = custom_table.sql_table_create_string() + + # step 1 - convert question into sql + + if not table_schema: + logging.warning("LLMfx - query_db - could not identify table schema - can not proceed") + return -1 + + # run inference with query and table schema to get SQL query response + response = self.sql(query, table_schema) + + # step 2 - run query + sql_query = response["llm_response"] + self.sql_query = sql_query + + # basic sql verification checker + confirmation, self.sql_query = self.sql_checker(self.sql_query, custom_sql_checker=custom_sql_checker) + + if not confirmation: + logger.warning(f"LLMfx - query_custom_db - sql query generated appears to be potentially unsafe - " + f"{self.sql_query} so not moving ahead with query.") + + empty_result = {"step": self.step, "tool": "sql", "db_response": [], + "sql_query": self.sql_query + "-NOT_EXECUTED", + "query": query, "db": db, "work_item": table_schema} + + self.research_list.append(empty_result) + + return empty_result + + # initial journal update + journal_update = f"executing research call - executing query on db\n" + journal_update += f"\t\t\t\t -- db - {db}\n" + journal_update += f"\t\t\t\t -- sql_query - {self.sql_query}" + self.write_to_journal(journal_update) + + db_output = custom_table.custom_lookup(self.sql_query) + + output = [] + db_response = list(db_output) + + for rows in db_response: + output.append(rows) + + result = {"step": self.step, "tool": "sql", "db_response": output, "sql_query": self.sql_query, + "query": query,"db": db, "work_item": table_schema} + + self.research_list.append(result) + + # start journaling update + journal_update = f"executing research - getting response - sql\n" + journal_update += f"\t\t\t\t -- result - {str(output)}" + # journal_update += f"\t\t\t\t -- output type - text" + + self.write_to_journal(journal_update) + # end journaling + + return result + + def query_db(self, query, table=None, table_schema=None, db=None, db_name=None, + custom_sql_checker=None): + + """ Executes two steps - converts input query into SQL, and then executes the SQL query on the DB. """ + + sql_db = SQLTables(db=db, db_name=db_name) + + if not table_schema: + if table: + table_schema = sql_db.get_table_schema(table) + + # step 1 - convert question into sql + + if not table_schema: + logging.warning("LLMfx - query_db - could not identify table schema - can not proceed") + return -1 + + # run inference with query and table schema to get SQL query response + response = self.sql(query, table_schema) + + # step 2 - run query + sql_query = response["llm_response"] + self.sql_query = sql_query + sql_db_name = sql_db.db_file + + # basic sql safety check + confirmation, self.sql_query = self.sql_checker(self.sql_query, custom_sql_checker=custom_sql_checker) + + if not confirmation: + logger.warning(f"LLMfx - query_db - sql query generated appears to be potentially unsafe - " + f"{self.sql_query} so not moving ahead with query.") + + empty_result = {"step": self.step, "tool": "sql", "db_response": [], + "sql_query": self.sql_query + "-NOT_EXECUTED", + "query": query, "db": db, "work_item": table_schema} + + self.research_list.append(empty_result) + + return empty_result + + # initial journal update + journal_update = f"executing research call - executing query on db\n" + journal_update += f"\t\t\t\t -- db - {sql_db_name}\n" + journal_update += f"\t\t\t\t -- sql_query - {self.sql_query}" + self.write_to_journal(journal_update) + + db_output = sql_db.query_db(self.sql_query) + + output = [] + db_response = list(db_output) + + for rows in db_response: + output.append(rows) + + result = {"step": self.step, "tool": "sql", "db_response": output, "sql_query": self.sql_query, + "query": query,"db": sql_db_name, "work_item": table_schema} + + self.research_list.append(result) + + # start journaling update + journal_update = f"executing research - getting response - sql\n" + journal_update += f"\t\t\t\t -- result - {str(output)}" + # journal_update += f"\t\t\t\t -- output type - text" + + self.write_to_journal(journal_update) + # end journaling + + return result + + def token_comparison (self, value_string, context): + + """ Utility function to perform token-level comparison in llm_response with input source materials. """ + + # note: this is a more limited version of the QualityCheck tools used in Prompt class + + c = CorpTokenizer(remove_stop_words=True, remove_numbers=False, + one_letter_removal=True, remove_punctuation=False) + + llm_response_tokens = c.tokenize(value_string) + context_tokens = c.tokenize(context) + + # iterate thru each key point and analyze comparison match + matched = [] + unmatched = [] + + for i, tok in enumerate(llm_response_tokens): + + if tok.endswith("."): + tok = tok[:-1] + + if tok.endswith(";"): + tok = tok[:-1] + + tok = re.sub("[,();$\"\n\r\t\u2022\u201c\u201d]", "", tok) + + if len(tok) > 0: + + match_found = False + + for j, etoks in enumerate(context_tokens): + + if etoks.endswith("."): + etoks = etoks[:-1] + + if etoks.endswith(";"): + etoks = re.sub("[(),;$\n\r\t\"\u2022\u201c\u201d]", "", etoks) + + if tok == etoks: + # found matching token + match_found = True + matched.append(tok) + break + + if not match_found: + unmatched.append(tok) + + # match_percent = 0.0 + match_percent = "{0:.1f}%".format(0.0) + match_fr = 0.0 + + if (len(matched) + len(unmatched)) > 0: + + match_fr = len(matched) / (len(matched) + len(unmatched)) + + if match_fr > 1.0: + match_fr = 1.0 + + match_percent = "{0:.1f}%".format((match_fr * 100)) + + comparison_stats = {"percent_display": match_percent, + "confirmed_words": matched, + "unconfirmed_words": unmatched, + "verified_token_match_ratio": match_fr, + } + + return comparison_stats + + def fx_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="", + function=None, endpoint_base=None, api_key=None, get_logits=False): + + # send to api agent server + + import ast + import requests + + if endpoint_base: + self.api_endpoint = endpoint_base + + if api_key: + # e.g., "demo-test" + self.api_key = api_key + + if not params: + model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type] + mc = ModelCatalog().lookup_model_card(model_name) + if "primary_keys" in mc: + params = mc["primary_keys"] + + url = self.api_endpoint + "{}".format("/agent") + output_raw = requests.post(url, data={"model_name": model_name, "api_key": self.api_key, "tool_type": tool_type, + "function": function, "params": params, "max_output": 50, + "temperature": 0.0, "sample": False, "prompt": prompt, + "context": context, "get_logits": True}) + + try: + # output = ast.literal_eval(output_raw.text) + output = json.loads(output_raw.text) + if "logits" in output: + logits = ast.literal_eval(output["logits"]) + self.agent_writer.write(f"logits: {logits}") + output["logits"] = logits + if "output_tokens" in output: + ot_int = [int(x) for x in output["output_tokens"]] + output["output_tokens"] = ot_int + + # need to clean up logits + except: + logging.warning("warning: api inference was not successful") + output = {} + + self.agent_writer.write(f"TEST: executed Agent call over API endpoint - {model_name} - {function} - {output}") + + return output + + +class SQLTables: + + """ SQLTables is a class for creating and accessing external SQL data, primarily as a resource that is + accessible via Text2SQL programmatic inferences. + + This is an **experimental** feature, and currently supports only use of SQLite, configured as a separate + local file-based DB, e.g., sqlite-experimental.db + + Use of this class will create a separate sqlite_experimental.db per the configs in SQLiteConfig + + Please note that the CustomTables class in llmware.resources provides a superset of this functionality, and + offers support for Postgres, in addition to SQLite. This class is provided for a 'fast example' but + generally we would recommend using CustomTables for more complex use cases. + + """ + + def __init__(self, db=None, db_name=None, experimental=True): + + self.db = "sqlite" + + # check for llmware path & create if not already set up, e.g., "first time use" + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + logger.info("SQLTables - Setting up LLMWare Workspace.") + + # default config for "db_experimental" = "sqlite_experimental.db" + self.db_name = SQLiteConfig().get_config("db_experimental") + + if experimental: + self.db_file = SQLiteConfig().get_uri_string_experimental_db() + logging.info("update: connecting to experimental sqlite db - %s", self.db_file) + + else: + self.db_file = SQLiteConfig().get_uri_string() + logging.info("warning: connecting to main sqlite db - %s", self.db_file) + + self.conn = sqlite3.connect(self.db_file) + + self.tables = [] + + def get_table_schema(self,table_name): + + """ Lookup of table_schema for an input table_name - outputs 'create table schema string' that can + be used directly as context in a text2sql inference """ + + table_schema = "" + + sql_query = f"SELECT * FROM sqlite_master WHERE type = 'table' AND name = '{table_name}';" + + table_schema_row = self.conn.cursor().execute(sql_query) + table_schema_row = list(table_schema_row) + + if len(table_schema_row) > 0: + table_schema = table_schema_row[0][4] + + return table_schema + + def get_column_names(self, table_name): + + """ Gets the column names from a table, and provides a list as output. """ + + column_names = [] + + sql_query_pragma = "PRAGMA table_info('{}')".format(table_name) + column_info = self.conn.cursor().execute(sql_query_pragma) + + for entries in column_info: + column_names.append(entries[1]) + + return column_names + + def query_db(self, sql_query): + + """ Executes a query directly on database """ + + # note: security and access are left to the user to manage + + try: + result = self.conn.cursor().execute(sql_query) + except: + logging.warning("update: query generated error - not successful - %s", sql_query) + + # if sql query generates error, then an empty result is returned + result = [] + + return result + + def delete_experimental_db(self, confirm_delete=False): + + """ Deletes the experimental db """ + + # delete db and start fresh + if confirm_delete: + shutil.rmtree(self.db_file) + logging.warning("update: deleted sqlite experimental db - %s ", self.db_file) + + return True + + def delete_table(self, table_name, confirm_delete=False): + + """ Deletes a table on the experimental db """ + + if confirm_delete: + + sql_instruction = f"DROP TABLE {table_name};" + results = self.conn.cursor().execute(sql_instruction) + self.conn.commit() + logging.warning("update: delete sqlite experimental db - table - %s ", table_name) + + return 0 + + def register_table(self, sql_table_create): + self.tables.append(sql_table_create) + return self.tables + + def reset_tables(self): + self.tables = [] + return True + + def table_exists_check(self, table_name): + + """Checks if table exists - true if exists, false if does not exist. """ + + sql_query = f"SELECT * FROM sqlite_master WHERE type = 'table' AND name = '{table_name}';" + + results = self.conn.cursor().execute(sql_query) + + if len(list(results)) > 0: + table_exists = True + else: + table_exists = False + + return table_exists + + def load_csv(self, fp, fn): + + """ Opens CSV file at folder_path fp and file_name fn and returns array-like output in memory """ + + in_path = os.path.join(fp,fn) + + # csv encoding can vary - utf-8-sig and errors='ignore' seems to be the most resilient for wide range of csv + record_file = open(in_path, encoding='utf-8-sig',errors='ignore') + c = csv.reader(record_file, dialect='excel', doublequote=False, delimiter=',') + output = [] + for lines in c: + output.append(lines) + record_file.close() + + return output + + def create_new_table(self, output, table_name): + + """ Creates a new table, deriving the column names from an implied header row in the output, + and a sniff test on the value types. """ + + col_names = [] + + if len(output) > 1: + header_row = output[0] + test_row = output[1] + + keys_list = "(" + + sql_create_table = f"CREATE TABLE {table_name} (" + for i, entry in enumerate(header_row): + col_name = re.sub("[\xfe\xff]","",entry) + try: + #TODO: build more robust type checking, e.g., float/decimal/currency + test_int = int(test_row[i]) + type="integer" + except: + type="text" + + col_names.append(col_name) + + keys_list += col_name + ", " + + sql_create_table += col_name + " " + type + ", " + + if sql_create_table.endswith(", "): + sql_create_table = sql_create_table[:-2] + + sql_create_table += " )" + + if keys_list.endswith(", "): + keys_list = keys_list[:-2] + + keys_list += " )" + + self.conn.cursor().execute(sql_create_table) + + return col_names + + def insert_new_row(self, table_name, keys_list, new_row): + + """ Inserts a new row into table. """ + + col_names = "(" + for cols in keys_list: + col_names += cols + ", " + if col_names.endswith(", "): + col_names = col_names[:-2] + col_names += ")" + + values_list = "(" + for j in range(0, len(new_row)): + values_list += "$" + str(j + 1) + ", " + + if values_list.endswith(", "): + values_list = values_list[:-2] + + values_list += ")" + + new_record = f"INSERT INTO {table_name} {col_names} VALUES {values_list};" + + logging.info("update: inserting new_record - %s ", new_record) + + self.conn.cursor().execute(new_record, new_row) + + return True + + def create_new_table_from_csv(self,fp=None, fn=None, table_name=None): + + """ Designed for rapid prototyping - input is a well-formed csv file with assumed header row with + each entry representing a column name, and well-formed rows. """ + + # load csv + output = self.load_csv(fp,fn) + + # check if table exists + if not self.table_exists_check(table_name): + + logging.info("update: table does not exist - so creating") + # need to build the table + column_names = self.create_new_table(output, table_name) + logging.info("update: table created - column names - %s ", column_names) + + else: + logging.info("update: table exists - getting column names") + column_names = self.get_column_names(table_name) + + # insert records + + new_record = "" + for i in range(1, len(output)): + + self.insert_new_row(table_name,column_names,output[i]) + + self.conn.commit() + self.conn.close() + + logging.info("update: done inserting records into new table") + + return 0 + + + + + diff --git a/llmware/configs.py b/llmware/configs.py new file mode 100644 index 0000000..6a41352 --- /dev/null +++ b/llmware/configs.py @@ -0,0 +1,1324 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The configs module implements the configuration logic using classes for llmware. + +The implementation includes the central llmware config class LLMWareConfig, and the config classes for all +supported text index databases and vector databases. +""" + +import os +import platform +import logging + +try: + from colorama import Fore + COLOR_WHITE = Fore.WHITE + COLOR_RESET = Fore.RESET + COLOR_RED = Fore.RED + COLOR_YELLOW = Fore.YELLOW + COLOR_GREEN= Fore.GREEN + COLOR_BLUE = Fore.BLUE +except: + COLOR_WHITE = "" + COLOR_RESET = "" + COLOR_RED = "" + COLOR_YELLOW = "" + COLOR_GREEN = "" + COLOR_BLUE = "" + + +class CustomFormatter(logging.Formatter): + + """ CustomFormatter - Configuration of global logging formatting - WIP. """ + + format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" + + FORMATS = { + logging.DEBUG: COLOR_GREEN + format + COLOR_RESET, + logging.INFO: COLOR_WHITE + format + COLOR_RESET, + logging.WARNING: COLOR_YELLOW + format + COLOR_YELLOW, + logging.ERROR: COLOR_RED + format + COLOR_RESET, + logging.CRITICAL: COLOR_RED + format + COLOR_RESET, + 25: COLOR_BLUE + format + COLOR_RESET + } + + def format(self, record): + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt) + return formatter.format(record) + + +class LLMWareConfig: + + """LLMWare global configuration object - use set/get to update """ + + if platform.system() == "Windows": + + _base_fp = {"home_path": os.environ.get("USERPROFILE"), + "llmware_path_name": "llmware_data" + os.sep} + else: + _base_fp = {"home_path": os.environ.get("HOME"), + "llmware_path_name": "llmware_data" + os.sep} + + _fp = {"model_repo_path_name": "model_repo" + os.sep, + "library_path_name": "accounts" + os.sep, + "input_path_name": "input_channel" + os.sep, + "parser_path_name": "parser_history" + os.sep, + "query_path_name": "query_history" + os.sep, + "prompt_path_name": "prompt_history" + os.sep, + "tmp_path_name": "tmp" + os.sep} + + # note: two alias for postgres vector db - "postgres" and "pg_vector" are the same + _supported = {"vector_db": ["chromadb", "neo4j", "milvus", "pg_vector", "postgres", "redis", "pinecone", "faiss", "qdrant", "mongo_atlas","lancedb"], + "collection_db": ["mongo", "postgres", "sqlite"], + "table_db": ["postgres", "sqlite"]} + + # change 0.4.0: default collection_db set to "sqlite" + _conf = {"collection_db": "sqlite", + "vector_db": "milvus", + "table_db": "sqlite", + "debug_mode": 0, + "llmware_sample_files_bucket": "llmware-sample-docs", + "llmware_public_models_bucket": "llmware-public-models", + "shared_lib_path": os.path.join(os.path.dirname(os.path.realpath(__file__)), "lib"), + "logging_level": logging.WARNING, + "logging_format": COLOR_WHITE + '%(levelname)-4s: %(message)s' + COLOR_RESET, + "logging_level_by_module": {"llmware.embeddings": 20, "llmware.models": 20, "llmware.agents":20, + "llmware.prompts": 20, "llmware.resources": 20, + "llmware.setup": 20, "llmware.parsers": 20}, + "agent_writer_mode": "screen", + "agent_log_file": "agent_log.txt", + "model_register": {"module": "llmware.models", "method": "register"}, + "model_post_init": {"module": "llmware.models", "method": "post_init"}, + "model_validate": {"module": "llmware.models", "method": "validate"}, + "model_preview": {"module": "llmware.models", "method": "preview"}, + "model_fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "model_router": {"module": "llmware.models", "method": "route_optimizer"}, + "apply_model_load_router": False, + "apply_default_fetch_override": False + } + + @classmethod + def get_config(cls,name): + """Get config value by key""" + + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls,name, value): + """Set config value by key""" + cls._conf[name] = value + + @classmethod + def get_home(cls): + """Get home directory path""" + return cls._base_fp["home_path"] + + @classmethod + def set_home(cls, new_value): + """Set home directory path""" + cls._base_fp["home_path"] = new_value + + @classmethod + def set_llmware_path_name(cls, new_value): + """Set main path name for llmware data path""" + cls._base_fp["llmware_path_name"] = new_value + + @classmethod + def get_fp_name(cls,file_path): + """Get file path from configs""" + if file_path in cls._fp: + return cls._fp[file_path] + raise ConfigKeyException(file_path) + + @classmethod + def set_fp_name(cls,file_path, new_value): + """Set file path in configs""" + if file_path in cls._fp: + cls._fp.update({file_path, new_value}) + + @classmethod + def get_llmware_path(cls): + """Get llmware absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"], cls._base_fp["llmware_path_name"]) + + @classmethod + def get_library_path(cls): + """Get library absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"], cls._base_fp["llmware_path_name"], cls._fp["library_path_name"]) + + @classmethod + def get_model_repo_path(cls): + """Get model repo absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"],cls._base_fp["llmware_path_name"], cls._fp["model_repo_path_name"]) + + @classmethod + def get_input_path(cls): + """Get input absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"], cls._base_fp["llmware_path_name"], cls._fp["input_path_name"]) + + @classmethod + def get_parser_path(cls): + """Get parser absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"],cls._base_fp["llmware_path_name"], cls._fp["parser_path_name"]) + + @classmethod + def get_query_path(cls): + """Get query absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"],cls._base_fp["llmware_path_name"], cls._fp["query_path_name"]) + + @classmethod + def get_prompt_path(cls): + """Get prompt absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"], cls._base_fp["llmware_path_name"],cls._fp["prompt_path_name"]) + + @classmethod + def get_tmp_path(cls): + """Get tmp absolute folder directory path""" + return os.path.join(cls._base_fp["home_path"], cls._base_fp["llmware_path_name"],cls._fp["tmp_path_name"]) + + @classmethod + def get_path(cls, name): + """Get absolute folder path by name""" + + if name+"_name" in cls._fp: + return os.path.join(cls._base_fp["home_path"], cls._base_fp["llmware_path_name"], + cls._fp[name+"_name"]) + + raise LLMWareException(message=f"LLMWareConfig - get_path - could not find home path - {name}") + + @classmethod + def get_active_db(cls): + + """ Returns the current selected default database for Library text collections """ + + return cls._conf["collection_db"] + + @classmethod + def set_active_db(cls, new_db): + + """ Sets the default database for Library text collections """ + + if new_db in cls._supported["collection_db"]: + cls._conf["collection_db"] = new_db + else: + raise LLMWareException(message=f"LLMWareConfig - set_active_db - selected " + f"db is not supported - {new_db}") + + @classmethod + def get_vector_db(cls): + + """ Gets the default vector database selection """ + + return cls._conf["vector_db"] + + @classmethod + def set_vector_db(cls, vector_db_name): + + """ Sets the default vector database """ + + if vector_db_name in cls._supported["vector_db"]: + cls._conf["vector_db"] = vector_db_name + else: + raise LLMWareException(message=f"LLMWareConfig - set_vector_db - " + f"selected db is not supported - {vector_db_name}") + + @classmethod + def get_table_db(cls): + + """ Gets the default table (SQL) database """ + + return cls._conf["table_db"] + + @classmethod + def set_table_db(cls, table_db_name): + + """ Sets the default table (SQL) database """ + + if table_db_name in cls._supported["table_db"]: + cls._conf["table_db"] = table_db_name + else: + raise LLMWareException(message=f"LLMWareConfig - set_table_db - " + f"selected db is not supported - {table_db_name}") + + @classmethod + def get_supported_vector_db(cls): + return cls._supported["vector_db"] + + @classmethod + def get_supported_collection_db(cls): + return cls._supported["collection_db"] + + @classmethod + def get_supported_table_db(cls): + return cls._supported["table_db"] + + @classmethod + def setup_llmware_workspace (cls): + + """Set up llmware main working folder directory""" + + # create file structure - configured through use of env variable ["HOME"] + home_path = cls._base_fp["home_path"] + + if not os.path.exists(home_path): + raise LLMWareException(message=f"LLMWareConfig - get_path - could not find home path - {home_path}") + + llmware_path = cls.get_llmware_path() + if not os.path.exists(llmware_path): + os.mkdir(llmware_path) + + library_path = cls.get_library_path() + if not os.path.exists(library_path): + os.mkdir(library_path) + + input_path = cls.get_input_path() + if not os.path.exists(input_path): + os.mkdir(input_path) + + model_repo_path = cls.get_model_repo_path() + if not os.path.exists(model_repo_path): + os.mkdir(model_repo_path) + + parser_path = cls.get_parser_path() + if not os.path.exists(parser_path): + os.mkdir(parser_path) + + query_path = cls.get_query_path() + if not os.path.exists(query_path): + os.mkdir(query_path) + + prompt_path = cls.get_prompt_path() + if not os.path.exists(prompt_path): + os.mkdir(prompt_path) + + tmp_path = cls.get_tmp_path() + if not os.path.exists(tmp_path): + os.mkdir(tmp_path) + + # set 'open' read/write directory permissions, e.g., chmod 777 + os.chmod(library_path, 0o777) + os.chmod(input_path, 0o777) + os.chmod(model_repo_path, 0o777) + os.chmod(parser_path, 0o777) + os.chmod(query_path, 0o777) + os.chmod(prompt_path, 0o777) + os.chmod(tmp_path, 0o777) + + return 0 + + @classmethod + def create_new_account(cls, account_name): + + """Sets up a new secondary account file structure""" + + # Useful in deploying llmware in multi-account and multi-user applications + # note: assumes account user permissions to be implemented in calling application + + library_path = cls.get_library_path() + new_account_path = os.path.join(library_path, account_name) + os.mkdir(new_account_path) + + return 0 + + @classmethod + def get_db_uri_string(cls): + + """ Retrieves the db_uri_string for the current active default text collection database """ + + active_db = cls.get_active_db() + + uri_string = None + + if active_db == "mongo": uri_string = MongoConfig.get_uri_string() + if active_db == "postgres": uri_string = PostgresConfig.get_uri_string() + if active_db == "sqlite": uri_string = SQLiteConfig.get_uri_string() + + return uri_string + + @classmethod + def get_db_configs(cls): + + """ Gets the db configs for the selected default text collection database """ + + active_db = cls.get_active_db() + + configs = {} + + if active_db == "mongo": configs = MongoConfig.get_db_configs() + if active_db == "postgres": configs = PostgresConfig.get_db_configs() + if active_db == "sqlite": configs = SQLiteConfig.get_db_configs() + + return configs + + @classmethod + def get_db_user_name(cls): + + """ Get the db user name for the default db """ + + active_db = cls.get_active_db() + + user_name = "" + + if active_db == "mongo": user_name = MongoConfig.get_user_name() + if active_db == "postgres": user_name = PostgresConfig.get_user_name() + if active_db == "sqlite": user_name = SQLiteConfig.get_user_name() + + return user_name + + @classmethod + def get_db_pw(cls): + + """ Get the db password for the default db """ + + active_db = cls.get_active_db() + + pw = "" + + if active_db == "mongo": pw = MongoConfig.get_db_pw() + if active_db == "postgres": pw = PostgresConfig.get_db_pw() + if active_db == "sqlite": pw = SQLiteConfig.get_db_pw() + + return pw + + @classmethod + def get_logging_level(cls): + return cls._conf["logging_level"] + + @classmethod + def set_logging_level(cls, log_level_str): + + log_level_str = log_level_str.upper() + + accepted_levels = ["INFO", "DEBUG", "WARNING", "ERROR","CRITICAL"] + if log_level_str not in accepted_levels: + raise ConfigKeyException(f"Exception: proposed logging level does not correspond to a " + f"recognized level, e.g., {accepted_levels}") + + if log_level_str == "INFO": new_level = logging.INFO + elif log_level_str == "DEBUG": new_level = logging.DEBUG + elif log_level_str == "WARNING": new_level = logging.WARNING + elif log_level_str == "ERROR": new_level = logging.ERROR + elif log_level_str == "CRITICAL": new_level = logging.CRITICAL + else: + new_level = logging.DEBUG + + cls._conf["logging_level"] = new_level + + return new_level + + @classmethod + def get_logging_format(cls): + return cls._conf["logging_format"] + + @classmethod + def set_logging_format(cls, formatting_str): + cls._conf["logging_format"] = formatting_str + return True + + @classmethod + def get_logging_level_by_module(cls, module): + if module in cls._conf["logging_level_by_module"]: + return cls._conf["logging_level_by_module"][module] + else: + raise ConfigKeyException(module) + + @classmethod + def set_logging_level_by_module(cls, module, level): + + if module in cls._conf["logging_level_by_module"]: + cls._conf["logging_level_by_module"][module] = level + else: + cls._conf["logging_level_by_module"].update({module: level}) + + return cls._conf["logging_level_by_module"] + + @classmethod + def get_agent_writer_mode(cls): + return cls._conf["agent_writer_mode"] + + @classmethod + def set_agent_writer_mode(cls, mode): + + if mode in ["screen", "file", "off"]: + cls._conf["agent_writer_mode"] = mode + else: + raise ConfigKeyException(mode) + + return cls._conf["agent_writer_mode"] + + @classmethod + def get_agent_log_file(cls): + return cls._conf["agent_log_file"] + + @classmethod + def set_agent_log_file(cls, log_file_name): + cls._conf["agent_log_file"] = log_file_name + return cls._conf["agent_log_file"] + + +class VectorDBRegistry: + + """ Registry of supported Vector DBs, and the class module used to interface with the DB. """ + + vector_db_list = {"milvus": {"module": "llmware.embeddings", "class": "EmbeddingMilvus"}, + "chromadb": {"module": "llmware.embeddings", "class": "EmbeddingChromaDB"}, + "qdrant": {"module": "llmware.embeddings", "class": "EmbeddingQdrant"}, + "postgres": {"module": "llmware.embeddings", "class": "EmbeddingPGVector"}, + "pg_vector": {"module": "llmware.embeddings", "class": "EmbeddingPGVector"}, + "redis": {"module": "llmware.embeddings", "class": "EmbeddingRedis"}, + "neo4j": {"module": "llmware.embeddings", "class": "EmbeddingNeo4j"}, + "lancedb": {"module": "llmware.embeddings", "class": "EmbeddingLanceDB"}, + "faiss": {"module": "llmware.embeddings", "class": "EmbeddingFAISS"}, + "pinecone": {"module": "llmware.embeddings", "class": "EmbeddingPinecone"}, + "mongo_atlas": {"module": "llmware.embeddings", "class": "EmbeddingMongoAtlas"} + } + @classmethod + def get_vector_db_list(cls): + """ List current view of implemented supported vector db for embeddings. """ + return cls.vector_db_list + + @classmethod + def add_vector_db(cls, db_name, vector_db_class, module="llmware.embeddings"): + """ Adds a vector db including the module and class. """ + new_entry = {db_name: {"module": module, "class": vector_db_class}} + cls.vector_db_list.update(new_entry) + return True + + +logging.basicConfig(format=LLMWareConfig().get_logging_format(), level=LLMWareConfig().get_logging_level()) + + +class ONNXConfig: + + """ Configuration object for ONNXRuntime and ONNXRuntime Genai - these parameters are consumed by + the ONNXGenerativeModel class in module llmware.models. In most cases, the parameters do not + require attention, but provided for more options to adapt to particular environments and use cases. """ + + # note: breaking changes in the onnxruntime_genai api starting in version 0.6, which was released in + # February 2025 - if you pip install, you should get version 0.6 + # if using an older version of onnxruntime_genai, then set the legacy flag to True + + _conf = {"version": "0.6", + "legacy": False} + + @classmethod + def get_config(cls, param): + return cls._conf[param] + + @classmethod + def set_config(cls, param, value): + cls._conf[param] = value + + @classmethod + def get_legacy_flag(cls): + return cls._conf["legacy"] + + @classmethod + def set_legacy_flag(cls, boolean_val): + cls._conf["legacy"] = boolean_val + return boolean_val + + +class OVConfig: + + """ Configuration object for OpenVino - these parameters are consumed by the + OVGenerativeModel class in module llmware.models. In most cases, the parameters + do not require attention, but provided for more options for performance tuning with + GPU deployment in particular. """ + + _conf = {"device": "GPU", + "use_ov_tokenizer": False, + "generation_version": "ov_genai_pip", + "use_gpu_if_available": True, + "cache": True, + "cache_with_model": True, + "cache_custom_path": "", + "apply_performance_hints": True, + "verbose_mode": False, + "get_token_counts": True + } + + _supported_hints = ["MODEL_PRIORITY", "GPU_HOST_TASK_PRIORITY", + "GPU_QUEUE_THROTTLE", "GPU_QUEUE_PRIORITY"] + + # this is a subset of useful GPU performance hints - will expand options over time + + _gpu_hints = { + "MODEL_PRIORITY": "HIGH", + "GPU_HOST_TASK_PRIORITY": "HIGH", + "GPU_QUEUE_THROTTLE": "HIGH", + "GPU_QUEUE_PRIORITY": "HIGH" + } + + @classmethod + def get_config(cls, param): + return cls._conf[param] + + @classmethod + def set_config(cls, param, value): + cls._conf[param]= value + + @classmethod + def get_gpu_hints(cls): + return cls._gpu_hints + + @classmethod + def set_gpu_hint(cls, param, value): + + # will add safety checks for type - most in form of "HIGH" | "MEDIUM" | "LOW" + # for more information, please see OpenVino documentation + if param in cls._supported_hints: + cls._gpu_hints[param] = value + + @classmethod + def optimize_for_gpu(cls): + return cls._conf["use_gpu_if_available"] + + @classmethod + def generation_version(cls): + return cls._conf["generation_version"] + + +class MilvusConfig: + + """Configuration object for Milvus""" + + _conf = {"host": os.environ.get("MILVUS_HOST", "localhost"), + "port": os.environ.get("MILVUS_PORT", 19530), + "db_name": os.environ.get("MILVUS_DB", "default"), + "partitions": [], + + # new attributes to support embedded milvus lite + "lite": False, + "lite_folder_path": LLMWareConfig().get_library_path(), + "lite_name": "milvus_lite.db", + } + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise "Key not found in configs" + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + +class MongoConfig: + """Configuration object for MongoDB""" + + _conf = {"db_uri": os.environ.get("COLLECTION_DB_URI", "mongodb://localhost:27017/"), + "user_name": "", + "pw": "", + "atlas_db_uri": "", + "db_name":""} + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + @classmethod + def get_uri_string(cls): + return cls._conf["db_uri"] + + @classmethod + def get_db_configs(cls): + configs = {} + for keys, values in cls._conf.items(): + configs.update({keys:values}) + return configs + + @classmethod + def get_user_name(cls): + return cls._conf["user_name"] + + @classmethod + def get_db_pw(cls): + return cls._conf["pw"] + + +class PostgresConfig: + + """Configuration object for Postgres DB""" + + _conf = {"host": os.environ.get("USER_MANAGED_PG_HOST", "localhost"), + "port": os.environ.get("USER_MANAGED_PG_PORT", 5432), + "db_name": os.environ.get("USER_MANAGED_PG_DB_NAME", "postgres"), + "user_name": os.environ.get("USER_MANAGED_PG_USER_NAME", "postgres"), + "pw": os.environ.get("USER_MANAGED_PG_PW", ""), + # to create full copy, set "postgres_schema" to "full" + "pgvector_schema": "vector_only"} + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + @classmethod + def get_uri_string(cls): + + port = cls._conf["port"] + host = cls._conf["host"] + db_name = cls._conf["db_name"] + + # canonical simple format of postgres uri string + input_collection_db_path = f"postgresql://postgres@{host}:{port}/{db_name}" + # print("update: postgres get_uri_string - ", input_collection_db_path) + + return input_collection_db_path + + @classmethod + def get_db_configs(cls): + configs = {} + for keys, values in cls._conf.items(): + configs.update({keys:values}) + return configs + + @classmethod + def get_user_name(cls): + return cls._conf["user_name"] + + @classmethod + def get_db_pw(cls): + return cls._conf["pw"] + + +class RedisConfig: + + """Configuration object for Redis""" + + _conf = {"host": os.environ.get("USER_MANAGED_REDIS_HOST", "localhost"), + "port": os.environ.get("USER_MANAGED_REDIS_PORT", 6379), + "user_name": "", + "pw": "", + "db_name": ""} + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + +class PineconeConfig: + + """Configuration object for Pinecone""" + + _conf = { + "pinecone_api_key": os.environ.get("USER_MANAGED_PINECONE_API_KEY"), + "pinecone_cloud": os.environ.get("USER_MANAGED_PINECONE_CLOUD"), + "pinecone_region": os.environ.get("USER_MANAGED_PINECONE_REGION") + } + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + +class LanceDBConfig: + + _conf = {'uri': '/tmp/lancedb/'} + + @classmethod + def get_config(cls,name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + +class SQLiteConfig: + + """Configuration object for SQLite""" + + _conf = {"host": os.environ.get("USER_MANAGED_SQLITE_HOST", "localhost"), + "port": os.environ.get("USER_MANAGED_SQLITE_PORT", 6333), + "sqlite_db_folder_path": LLMWareConfig().get_library_path(), + "user_name": "", + "pw": "", + "db_name": "sqlite_llmware.db", + # add new parameter for SQLTables + "db_experimental": "sqlite_experimental.db"} + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + @classmethod + def get_uri_string (cls): + """For SQLite the URI string is the local file with full absolute path""" + db_file = os.path.join(cls._conf["sqlite_db_folder_path"], cls._conf["db_name"]) + return db_file + + # new method for SQLTables DB + @classmethod + def get_uri_string_experimental_db(cls): + """For SQLite the URI string is the local file with full absolute path""" + db_file = os.path.join(cls._conf["sqlite_db_folder_path"], cls._conf["db_experimental"]) + return db_file + # end method + + @classmethod + def get_db_configs(cls): + configs = {} + for keys, values in cls._conf.items(): + configs.update({keys:values}) + return configs + + @classmethod + def get_user_name(cls): + return cls._conf["user_name"] + + @classmethod + def get_db_pw(cls): + return cls._conf["pw"] + + +class QdrantConfig: + + """Configuration object for Qdrant""" + + _conf = { + "location": os.environ.get("USER_MANAGED_QDRANT_LOCATION", None), + "host": os.environ.get("USER_MANAGED_QDRANT_HOST", None), + "url": os.environ.get("USER_MANAGED_QDRANT_URL", None), + "path": os.environ.get("USER_MANAGED_QDRANT_PATH", None), + "port": os.environ.get("USER_MANAGED_QDRANT_PORT", 6333), + "grpc_port": os.environ.get("USER_MANAGED_QDRANT_GRPC_PORT", 6334), + "https": os.environ.get("USER_MANAGED_QDRANT_HTTPS", None), + "api_key": os.environ.get("USER_MANAGED_QDRANT_API_KEY", None), + "prefix": os.environ.get("USER_MANAGED_QDRANT_PREFIX", None), + "timeout": os.environ.get("USER_MANAGED_QDRANT_TIMEOUT", None), + "prefer_grpc": os.environ.get("USER_MANAGED_QDRANT_PREFER_GRPC", False), + "force_disable_check_same_thread": os.environ.get( + "USER_MANAGED_QDRANT_FORCE_DISABLE_CHECK_SAME_THREAD", False + ), + } + + @classmethod + def get_config(cls, name=None): + if not name: + return cls._conf + + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + +class AWSS3Config: + + """Configuration object for AWS S3""" + + _conf = {"access_key": os.environ.get("USER_MANAGED_S3_ACCESS_KEY", ""), + "secret_key": os.environ.get("USER_MANAGED_S#_SECRET_KEY", "")} + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + +class LLMWareTableSchema: + + # notes: + # 1. bigserial type for Postgres + # 2. "text" and "table" replaced with "text_block" and "table_block" in SQL DB for safety / reserved + + _block = {"_id": "bigserial", + "block_ID": "integer", + "doc_ID": "integer", + "content_type": "text", + "file_type": "text", + "master_index": "integer", + "master_index2": "integer", + "coords_x": "integer", + "coords_y": "integer", + "coords_cx": "integer", + "coords_cy": "integer", + "author_or_speaker": "text", + "added_to_collection": "text", + "file_source": "text", + "table_block": "text", + "modified_date": "text", + "created_date": "text", + "creator_tool": "text", + "external_files": "text", + "text_block": "text", + "header_text": "text", + "text_search": "text", + "user_tags": "text", + "special_field1": "text", + "special_field2": "text", + "special_field3": "text", + "graph_status": "text", + "dialog": "text", + "embedding_flags": "jsonb", + "PRIMARY KEY": "(_id)"} + + _library_card = {"library_name": "text", + "embedding": "json", + "knowledge_graph": "text", + "unique_doc_id": "integer", + "documents": "integer", + "blocks": "integer", + "images": "integer", + "pages": "integer", + "tables": "integer", + "account_name": "text", + "PRIMARY KEY": "(library_name)"} + + # used for basic tests of db connectivity and access + _simple_test = {"library_name": "text", + "hello_number": "integer"} + + _status = {"key": "text", + "summary": "text", + "start_time": "text", + "end_time": "text", + "total": "integer", + "current": "integer", + "units": "text", + "PRIMARY KEY": "(key)"} + + _parser_record = {"_id": "bigserial", + "job_id": "text", + "parser_type": "text", + "library_name": "text", + "account_name": "text", + "file_name": "text", + "message": "text", + "ocr_flag": "text", + "fail_flag": "text", + "time_stamp": "text", + "PRIMARY KEY": "(_id)"} + + _reserved_schema_names = ["block", "library", "status", "parser_record"] + + _custom_schema_names = [] + + _custom_schema = [] + + @classmethod + def get_block_schema(cls): + return cls._block + + @classmethod + def get_library_card_schema(cls): + return cls._library_card + + @classmethod + def get_status_schema(cls): + return cls._status + + @classmethod + def get_parser_table_schema(cls): + return cls._parser_record + + @classmethod + def register_custom_schema(cls, table_name, schema,replace=False): + + if table_name not in cls._custom_schema_names: + cls._custom_schema_names.append(table_name) + + if table_name not in cls._custom_schema: + cls._custom_schema.append({table_name: schema}) + else: + if replace: + cls._custom_schema.append({table_name: schema}) + + return cls._custom_schema + + @classmethod + def get_custom_tables(cls): + return cls._custom_schema_names + + @classmethod + def get_custom_schema(cls): + return cls._custom_schema + + +class Neo4jConfig: + """Configuration object for Neo4j""" + + _conf = { + 'uri': os.environ.get('NEO4J_URI', 'neo4j://localhost:7687'), + 'user': os.environ.get('NEO4J_USERNAME', 'neo4j'), + 'password': os.environ.get('NEO4J_PASSWORD', 'neo4j'), + 'database': os.environ.get('NEO4J_DATABASE', 'llmware') + } + + @classmethod + def get_db_configs(cls): + configs = {} + for keys, values in cls._conf.items(): + configs.update({keys:values}) + return configs + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + @classmethod + def get_uri_string(cls): + return cls._conf["uri"] + + @classmethod + def get_user_name(cls): + return cls._conf["user"] + + @classmethod + def get_db_pw(cls): + return cls._conf["password"] + + @classmethod + def get_database_name(cls): + return cls._conf["database"] + + +class ChromaDBConfig: + + """Configuration object for chroma. + + Update - v0.2.12 - default is set to use chroma as a file-based local persistent storage. + This is a change from the previous default which used chroma as an in-memory (ephemeral) store. + + Chroma can be used with or without (default) a client/server architecture. If it is used with a client/server + architecture, you have to set the authentication mechanism. The authentication mechanism can be either + username/password or token. + - env variable CHROMA_HOST is None -> not client/server mode (default), + - env variable CHROMA_HOST is set -> client/server mode + + If you want to use Chroma without the client/server architecture, the env variable CHROMA_HOST has to be + None (default). In this mode, you can choose between in-memory (also called ephemeral, non-persistent) and + persistent. + + Update: starting v0.2.12 - persistent path set as default to LLMWareConfigs().library_path(), which is the + same default path for a local sqlite.db + - to change to in-memory / non-persistent, set this config to None + - e.g., ChromaDBConfig().set_config("persistent_path", None) + + If you want to use Chroma in client/server mode, the env variable CHROMA_HOST needs to be set. In addition, + you have to set + - env variable CHROMA_SERVER_AUTH_PROVIDER, and + - env variable CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER + the value depends on the authentication mechanism you want to use. + + For more information, please visit https://docs.trychroma.com/getting-started + """ + + _conf = { + + # update - v0.2.12 -> collection on ChromaDB corresponds with the LLMWare library name + # 'collection': os.environ.get('CHROMA_COLLECTION', 'llmware'), + + # + # update - v0.2.12 -> by default, persistent path set to make chroma persistent. + # If this is None, then an in-memory only chroma instance will be created. + # + 'persistent_path': LLMWareConfig().get_library_path(), + + # + # Configs below are only relevant when chromadb is run in client/server mode. + # + + 'host': os.environ.get('CHROMA_HOST', None), + 'port': os.environ.get('CHROMA_PORT', 8000), + 'ssl': os.environ.get('CHROMA_SSL', False), + 'headers': os.environ.get('CHROMA_HEADERS', {}), + + # The provider decides whether we use authentication via username and password, or via a token. + # - For the username and password, this has to be set to chromadb.auth.basic.BasicAuthServerProvider + # - For the token, this has to be set to chromadb.auth.token.TokenAuthServerProvider + 'auth_provider': os.environ.get('CHROMA_SERVER_AUTH_PROVIDER', None), + + # The credential provider supplies the username and password or the token. This setting hence + # depends on the variable just above. + # - For the username and password, this has to be set to chromadb.auth.providers.HtpasswdFileServerAuthCredentialsProvider + # - For the token, this has to be set to chromadb.auth.token.TokenAuthServerProvider + 'auth_credentials_provider': os.environ.get('CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER', None), + + # Settings for authentication via username and password. + 'user': os.environ.get('CHROMA_USERNAME', 'admin'), + 'password': os.environ.get('CHROMA_PASSWORD', 'admin'), + 'auth_credentials_file': os.environ.get('CHROMA_SERVER_AUTH_CREDENTIALS_FILE', 'server.htpasswd'), + + # Settings for authentication via token. + 'auth_credentials': os.environ.get('CHROMA_SERVER_AUTH_CREDENTIALS', None), + 'auth_token_transport_header': os.environ.get('CHROMA_SERVER_AUTH_TOKEN_TRANSPORT_HEADER', None), + } + + @classmethod + def get_db_configs(cls): + configs = {} + for keys, values in cls._conf.items(): + configs.update({keys:values}) + return configs + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + @classmethod + def get_uri_string(cls): + return cls._conf["uri"] + + @classmethod + def get_user_name(cls): + return cls._conf["user"] + + @classmethod + def get_db_pw(cls): + return cls._conf["password"] + + @classmethod + def get_auth_provider(cls): + return cls._conf["auth_provider"] + + @classmethod + def get_auth_credentials_provider(cls): + return cls._conf["auth_credentials_provider"] + + @classmethod + def get_auth_credentials_file(cls): + return cls._conf["auth_credentials_file"] + + @classmethod + def get_auth_credentials(cls): + return cls._conf["auth_credentials"] + + @classmethod + def get_auth_token_transport_header(cls): + return cls._conf["auth_token_transport_header"] + + +class OpenAIConfig: + + """Configuration object for OpenAI - primarily for configuring Azure OpenAI credentials. + + Primary use is to setup an AzureOpenAI client that will be used in place of the standard + OpenAI client. + + Within LLMWare, the OpenAI model classes will check this config before creating a new OpenAI client. + + If there is a client already established in _conf["openai_client"], that client will be used in the + inference/embedding process. + + For example: + + # create your AzureOpenAI client + + from openai import AzureOpenAI + + client = AzureOpenAI( + azure_endpoint=os.getenv("AZURE_OPENAI_ENDPOINT"), + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + api_version="2024-08-01-preview") + + # add that client to the OpenAIConfig: + + OpenAIConfig().set_azure_client(client) + + """ + + _conf = {"openai_client": None, + "api_key": None, # placeholder / not used currently + "api_version": None, # placeholder / not used currently + "use_azure_endpoint": False} # placeholder / not used currently + + _azure_model_name_maps = { + } + + # useful mapping for Azure OpenAI default model names + # e.g., Azure removes the "." + _azure_strip_dot = True + + @classmethod + def get_config(cls, name): + if name in cls._conf: + return cls._conf[name] + raise ConfigKeyException(name) + + @classmethod + def set_config(cls, name, value): + cls._conf[name] = value + + @classmethod + def set_azure_client(cls, azure_client): + cls._conf["openai_client"] = azure_client + + @classmethod + def get_azure_client(cls): + return cls._conf["openai_client"] + + @classmethod + def get_azure_model_name(cls, model_name): + + """ Azure OpenAI default names remove '.' from OpenAI names. """ + + if cls._azure_strip_dot: + azure_name = model_name.replace(".","") + else: + azure_name = model_name + + return azure_name + + @classmethod + def set_azure_strip_dot(cls, setting): + if isinstance(setting,bool): + cls._azure_strip_dot = setting + + +class LLMWareException(Exception): + + """ Base exception class in LLMWare. """ + + __module__ = 'llmware' + + def __init__(self, message="An unspecified error occurred"): + super().__init__(message) + self.message = message + + +class DependencyNotInstalledException(LLMWareException): + + def __init__(self, required_library_dependency): + message = f"'{required_library_dependency}' needs to be installed to use this function." + super().__init__(message) + + +class ModelNotFoundException(LLMWareException): + + def __init__(self, model_name): + message = f"'{model_name}' could not be located" + super().__init__(message) + + +class ConfigKeyException(LLMWareException): + + def __init__(self, config_key): + message = f"'{config_key}' is not a valid configuration key." + super().__init__(message) + + +class ModuleNotFoundException(LLMWareException): + + def __init__(self, module_name): + message = (f"Module '{module_name}' could not be located. Please confirm the file path and extension. " + f"\n--There may be a missing binary, or an unsupported " + f"operating system platform.\n--Binaries are shipped in LLMWare for Mac (Metal), Windows (x86), " + f"and Linux (x86).\n--Please try re-installing and check documentation or raise issue " + f"at main Github repository at: https://www.github.com/llmware-ai/llmware.git.") + + super().__init__(message) + + +class GGUFLibNotLoadedException(LLMWareException): + + """ Exception raised when GGUF Lib back-end can not be loaded successfully. Exception tries to be + more helpful in sharing suggestions for potential causes and remedies. """ + + def __init__(self, module_name, os_platform, use_gpu, _lib_path, custom_path): + + # over time, may add more details by os_platform + + if custom_path: + + message = (f"GGUF lib from custom path - '{_lib_path}' could not be successfully loaded. Please " + f"check that the lib is a llama_cpp back-end binary. Assuming that it is a valid build," + f"then the most likely cause of the error is that the back-end binary was compiled " + f"with instructions not compatible with the current system.") + + else: + + if not use_gpu: + + message = (f"GGUF lib '{module_name}' could not be successfully loaded from shared library. This is " + f"most likely because the prepackaged binary does not match your OS configuration. " + f"LLMWare ships with 6 pre-built GGUF back-ends for Mac (Metal, Metal-no-acc), Windows (x86, CUDA), and " + f"Linux (x86, CUDA). These binaries depend upon low-level instruction capabilities provided" + f"by the processor and OS, and for Windows and Linux assume that AVX and AVX2 will be " + f"enabled. Useful debugging tips:\n--Ensure llmware 0.2.4+ installed, and if cloned from " + f"repository that all of the libs were fully updated" + f"\n--Linux - Ubuntu 20+ and GLIBC 2.31+ (will likely not run if GLIBC < 2.31); " + f"\n--Check CPU capabilities using `py-cpuinfo' library, which will generate a nice " + f"dictionary view of supported OS capabilities.\n" + f"--Raise an Issue on llmware github and please share the py-cpuinfo report for your system.") + else: + + message = (f"GGUF lib '{module_name}' for CUDA could not be loaded successfully, and an attempt to " + f"fall-back to the CPU based library also failed. To debug the CUDA availability, " + f"check the following:\n--`nvcc --version` to get the CUDA Driver version on your system." + f"\n--Win CUDA and Linux CUDA builds today require at least CUDA >= 12.1.\n" + f"--Pytorch is used to identify CUDA information - please check with the commands - " + f"`torch.cuda.is_available()` and `torch.version.cuda` that Pytorch is finding the " + f"right driver on your system.\n--Raise on Issue on llwmare github and share relevant " + f"system details on OS and CUDA drivers installed.") + + super().__init__(message) + + diff --git a/llmware/embeddings.py b/llmware/embeddings.py new file mode 100644 index 0000000..14fde08 --- /dev/null +++ b/llmware/embeddings.py @@ -0,0 +1,2729 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The embeddings module implements the supported vector databases. + +The common abstraction for all supported vector databases is the EmbeddingHandler class, which supports +creating a new embedding, as well as searching and deleting the vector index. The module also implements the +_EmbeddingUtils class, which provides a set of functions used by all vector database classes. +""" + +import os, logging, re, time, uuid, itertools +import numpy as np +from importlib import util +import importlib + +from llmware.configs import (LLMWareConfig, MongoConfig, MilvusConfig, PostgresConfig, RedisConfig, + PineconeConfig, QdrantConfig, Neo4jConfig, LanceDBConfig, ChromaDBConfig, VectorDBRegistry, + LLMWareException, DependencyNotInstalledException, ModelNotFoundException) + +from llmware.resources import CollectionRetrieval, CollectionWriter, Status +from llmware.util import Utilities + +""" By default, no vector db drivers are loaded into global program space unless and until they are invoked. Within +each embedding class handler, there is a check if GLOBAL_{VECTOR_DB}_IMPORT is False, and if so, then the module +is loaded, and the GLOBAL_{VECTOR_DB}_IMPORT is set to True. """ + +pymilvus = None +GLOBAL_PYMILVUS_IMPORT = False + +chromadb = None +GLOBAL_CHROMADB_IMPORT = False + +lancedb = None +GLOBAL_LANCEDB_IMPORT = False + +faiss = None +GLOBAL_FAISS_IMPORT = False + +neo4j = None +GLOBAL_NEO4J_IMPORT = False + +qdrant_client = None +GLOBAL_QDRANT_IMPORT = False + +pinecone = None +GLOBAL_PINECONE_IMPORT = False + +redis = None +GLOBAL_REDIS_IMPORT = False + +# pgvector requires import of both pgvector and psycopg +pgvector = None +GLOBAL_PGVECTOR_IMPORT = False + +psycopg = None +GLOBAL_PSYCOPG_IMPORT = False + +# used in mongo-atlas +pymongo = None +GLOBAL_PYMONGO_IMPORT = False + +logger = logging.getLogger(__name__) +log_level = LLMWareConfig().get_logging_level_by_module("llmware.embeddings") +logger.setLevel(level=log_level) + + +class EmbeddingHandler: + + """Provides an interface to all supported vector databases, which is used by the ``Library`` class. + + ``EmbeddingHandler`` is responsible for embedding-related interactions between a library and a vector + store. This includes creating, reading, updating, and deleting (CRUD) embeddings. The ``EmbeddingHandler``, + in addition, synchronizes the vector store with the text collection database, this includes incremental + updates to the embeddings. Finally, it also allows one library to have multiple embeddings. + + Parameters + ---------- + library : Library + The library with which the ``EmbeddingHandler`` interacts. + + Returns + ------- + embedding_handler : EmbeddingHandler + A new ``EmbeddingHandler`` object. + """ + + def __init__(self, library): + + self.supported_embedding_dbs = VectorDBRegistry().get_vector_db_list() + + self.library = library + + def create_new_embedding(self, embedding_db, model, doc_ids=None, batch_size=500): + + """ Creates new embedding - routes to correct vector db and loads the model and text collection """ + + embedding_class = self._load_embedding_db(embedding_db, model=model) + embedding_status = embedding_class.create_new_embedding(doc_ids, batch_size) + + if embedding_status: + if "embeddings_created" in embedding_status: + if embedding_status["embeddings_created"] > 0: + # only update if non-zero embeddings created + if "embedded_blocks" in embedding_status: + embedded_blocks = embedding_status["embedded_blocks"] + else: + embedded_blocks = -1 + logger.warning("update: embedding_handler - unable to determine if embeddings have " + "been properly counted and captured. Please check if databases connected.") + + self.library.update_embedding_status("yes", model.model_name, embedding_db, + embedded_blocks=embedded_blocks, + embedding_dims=embedding_status["embedding_dims"], + time_stamp=embedding_status["time_stamp"]) + + return embedding_status + + def search_index(self, query_vector, embedding_db, model, sample_count=10): + + """ Main entry point to vector search query """ + + # Need to normalize the query_vector. + # Sometimes it comes in as [[1.1,2.1,3.1]] (from Transformers) and sometimes as [1.1,2.1,3.1] + # We'll make sure it's the latter and then each Embedding Class will deal with it how it needs to + + if len(query_vector) == 1: + query_vector = query_vector[0] + + embedding_class = self._load_embedding_db(embedding_db, model=model) + return embedding_class.search_index(query_vector,sample_count=sample_count) + + def delete_index(self, embedding_db, model_name, embedding_dims): + + """ Deletes vector embedding - note: does not delete the underlying text collection """ + + embedding_class = self._load_embedding_db(embedding_db, model_name=model_name, + embedding_dims=embedding_dims) + embedding_class.delete_index() + self.library.update_embedding_status("delete", model_name, embedding_db, + embedded_blocks=0, delete_record=True) + + return 0 + + def _load_embedding_db(self, embedding_db, model=None, model_name=None, embedding_dims=None): + + """ Looks up and loads the selected vector database """ + + if not embedding_db in self.supported_embedding_dbs: + raise LLMWareException(message=f"EmbeddingHandler - load_embedding_db - " + f"selected embedding db is not supported - {embedding_db}") + + vdb = self.supported_embedding_dbs[embedding_db] + + # dynamically load the module/class for the specific embedding handler + vdb_module = vdb["module"] + vdb_class = vdb["class"] + vdb_module = importlib.import_module(vdb_module) + + if hasattr(vdb_module, vdb_class): + model_class = getattr(vdb_module, vdb_class) + + return model_class(self.library, model=model, model_name=model_name,embedding_dims=embedding_dims) + + else: + raise LLMWareException(message=f"Exception: could not find class implementation for {embedding_db}, which " + f"is expected at: {vdb_module} - {vdb_class}.") + + def generate_index_name(self, account_name, library_name, model_name, max_component_length=19): + + """ Creates a unique name for the vector index that concats library_name + model_name + account_name """ + + index_name = account_name + + # Remove non-alphanumerics from the remaining components and if still longer than the max, remove middle chars + for s in [library_name, model_name]: + s = re.sub(r'\W+', '', s) + if len(s) > max_component_length: + excess_length = len(s) - max_component_length + left_length = (len(s) - excess_length) // 2 + right_start = left_length + excess_length + index_name += s[:left_length] + s[right_start:] + + # Return the lowercase name: + return index_name.lower() + + +class _EmbeddingUtils: + + """Provides functions to vector stores, such as creating names for the text collection database as well + as creating names for vector such, and creating a summary of an embedding process. + + ``_EmbeddingUTils`` provides utilities used by all vector stores, especially in interaction and + synchronization with the underlying text collection database. In short, it has functions for + creating names, the text index, the embedding flag, the block curser, and the embedding summary. + + Parameters + ---------- + library_name : str, default=None + Name of the library. + + model_name : str, default=None + Name of the model. + + account_name : str, default=None + Name of the account. + + db_name : str, default=None + Name of the vector store. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_utils : _EmbeddingUtils + A new ``_EmbeddingUtils`` object. + """ + + def __init__(self, library_name=None, model_name=None, account_name=None,db_name=None, + embedding_dims=None): + + self.library_name = library_name + self.account_name = account_name + self.model_name = model_name + self.db_name = db_name + self.embedding_dims = embedding_dims + self.collection_key= None + self.collection_name= None + + def create_safe_collection_name(self): + + """ Creates concatenated safe name for collection """ + + converted_library_name = re.sub(r"[-@_.\/ ]", "", self.library_name).lower() + if len(converted_library_name) > 18: + converted_library_name = converted_library_name[0:18] + + converted_model_name = re.sub(r"[-@_.\/ ]", "", self.model_name).lower() + if len(converted_model_name) > 18: + # chops off the start of the model name if longer than 18 chars + starter = len(converted_model_name) - 18 + converted_model_name = converted_model_name[starter:] + + converted_account_name = re.sub(r"[-@_.\/ ]", "", self.account_name).lower() + if len(converted_model_name) > 7: + converted_account_name = converted_account_name[0:7] + + # create collection name here - based on account + library + model_name + self.collection_name = f"{converted_account_name}_{converted_library_name}_{converted_model_name}" + + return self.collection_name + + def create_db_specific_key(self): + + """ Creates db_specific key """ + + # will leave "-" and "_" in file path, but remove "@" and " " + model_safe_path = re.sub(r"[@ ]", "", self.model_name).lower() + self.collection_key = f"embedding_{self.db_name}_" + model_safe_path + + return self.collection_key + + def get_blocks_cursor(self, doc_ids = None): + + """ Retrieves a cursor from the text collection database that will define the scope of text chunks + to be embedded """ + + if not self.collection_key: + self.create_db_specific_key() + + cr = CollectionRetrieval(self.library_name, account_name=self.account_name) + num_of_blocks, all_blocks_cursor = cr.embedding_job_cursor(self.collection_key,doc_id=doc_ids) + + return all_blocks_cursor, num_of_blocks + + def generate_embedding_summary(self, embeddings_created): + + """ Common summary dictionary at end of embedding job """ + + if not self.collection_key: + self.create_db_specific_key() + + cr = CollectionRetrieval(self.library_name,account_name=self.account_name) + embedded_blocks = cr.count_embedded_blocks(self.collection_key) + + embedding_summary = {"embeddings_created": embeddings_created, + "embedded_blocks": embedded_blocks, + "embedding_dims": self.embedding_dims, + "time_stamp": Utilities().get_current_time_now()} + + return embedding_summary + + def update_text_index(self, block_ids, current_index): + + """ Update main text collection db """ + + for block_id in block_ids: + + cw = CollectionWriter(self.library_name, account_name=self.account_name) + cw.add_new_embedding_flag(block_id,self.collection_key,current_index) + + current_index += 1 + + return current_index + + def lookup_text_index(self, _id, key="_id"): + + """Returns a single block entry from text index collection with lookup by _id - returns a list, not a cursor""" + + cr = CollectionRetrieval(self.library_name, account_name=self.account_name) + block_cursor = cr.lookup(key, _id) + + return block_cursor + + def lookup_embedding_flag(self, key, value): + + """ Used to look up an embedding flag in text collection index """ + + # used specifically by FAISS index - which uses the embedding flag value as lookup + cr = CollectionRetrieval(self.library_name, account_name=self.account_name) + block_cursor = cr.embedding_key_lookup(key,value) + + return block_cursor + + def unset_text_index(self): + + """Removes embedding key flag for library, e.g., 'unsets' a group of blocks in text index """ + + cw = CollectionWriter(self.library_name, account_name=self.account_name) + cw.unset_embedding_flag(self.collection_key) + + return 0 + + +class EmbeddingMilvus: + + """ + ``EmbeddingMilvus`` implements the interface to the ``Milvus`` vector store. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_milvus : EmbeddingMilvus + A new ``EmbeddingMilvus`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + self.library = library + self.library_name = library.library_name + self.account_name = library.account_name + self.milvus_alias = "default" + + self.use_milvus_lite = MilvusConfig().get_config("lite") + + # confirm that pymilvus installed + + global GLOBAL_PYMILVUS_IMPORT + if not GLOBAL_PYMILVUS_IMPORT: + if util.find_spec("pymilvus"): + + try: + global pymilvus + pymilvus = importlib.import_module("pymilvus") + GLOBAL_PYMILVUS_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load pymilvus module.") + + else: + raise LLMWareException(message="Exception: need to import pymilvus to use this class.") + + # end dynamic import here + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + self.model=model + self.model_name=model_name + self.embedding_dims = embedding_dims + + # if model passed (not None), then use model name + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = self.model.embedding_dims + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="milvus", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + if self.use_milvus_lite: + + logger.info(f"update: EmbeddingHandler - Milvus - selecting 'lite' version. If you intend to use " + f"a server-based version of Milvus, please set: MilvusConfig().set_config('lite', False).") + + lite_path = MilvusConfig().get_config("lite_folder_path") + lite_db_name = MilvusConfig().get_config("lite_name") + + self.collection = pymilvus.MilvusClient(os.path.join(lite_path, lite_db_name)) + + # check if collection_name found in list of collections - load, if exists, else create new + if self.collection_name in self.collection.list_collections(): + self.collection.load_collection(self.collection_name) + else: + schema = self.collection.create_schema( + auto_id=False, + enable_dynamic_field=True, + ) + + # add fields to schema + schema.add_field(field_name="block_mongo_id", datatype=pymilvus.DataType.VARCHAR, is_primary=True, + max_length=30, auto_id=False) + schema.add_field(field_name="block_doc_id", datatype=pymilvus.DataType.INT64) + schema.add_field(field_name="embedding_vector", datatype=pymilvus.DataType.FLOAT_VECTOR, dim=self.embedding_dims) + + index_params = self.collection.prepare_index_params() + + # add index + index_params.add_index( + field_name="embedding_vector", + metric_type="L2", + ) + + self.collection.create_collection(collection_name=self.collection_name, + dimension=self.embedding_dims, + schema=schema, + index_params=index_params) + + else: + + # connect to Milvus server + + logger.info(f"update: EmbeddingHandler - Milvus - connecting to Milvus server instance. To use " + f"Milvus 'lite', set MilvusConfig().set_config('lite', True).") + + pymilvus.connections.connect(self.milvus_alias, + host=MilvusConfig.get_config("host"), + port=MilvusConfig.get_config("port"), + db_name=MilvusConfig.get_config("db_name")) + + if not pymilvus.utility.has_collection(self.collection_name): + fields = [ + pymilvus.FieldSchema(name="block_mongo_id", + dtype=pymilvus.DataType.VARCHAR, is_primary=True, max_length=30,auto_id=False), + pymilvus.FieldSchema(name="block_doc_id", dtype=pymilvus.DataType.INT64), + pymilvus.FieldSchema(name="embedding_vector", dtype=pymilvus.DataType.FLOAT_VECTOR, + dim=self.embedding_dims) + ] + + collection = pymilvus.Collection(self.collection_name, pymilvus.CollectionSchema(fields)) + index_params = { + "metric_type": "L2", + "index_type": "IVF_FLAT", + "params": {"nlist": 1024} + } + collection.create_index("embedding_vector", index_params) + + self.collection = pymilvus.Collection(self.collection_name) + + def create_new_embedding(self, doc_ids = None, batch_size=500): + + """ Create new embedding """ + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.account_name) + status.new_embedding_status(self.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + current_index = 0 + finished = False + + while not finished: + + block_ids, doc_ids, sentences = [], [], [] + + # Build the next batch + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + + # data model + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + + # Process the batch + vectors = self.model.embedding(sentences) + data = [block_ids, doc_ids, vectors] + + if self.use_milvus_lite: + + d=[] + for i, vec in enumerate(vectors): + new_row = {"block_mongo_id": block_ids[i], "block_doc_id": doc_ids[i], "embedding_vector": vec} + d.append(new_row) + + self.collection.insert(data=d, collection_name=self.collection_name) + + else: + self.collection.insert(data) + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + + status.increment_embedding_status(self.library_name, self.model_name, len(sentences)) + + # will add configuration options to show/display + logger.info(f"update: embedding_handler - Milvus - Embeddings Created: {embeddings_created} of {num_of_blocks}") + + if not self.use_milvus_lite: + self.collection.flush() + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - Milvus - embedding_summary - {embedding_summary}") + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + + if not self.use_milvus_lite: + self.collection.load() + + search_params = { + "field_name": "embedding_vector", + "metric_type": "L2", + "params": {"nprobe": 10} + } + + # TODO: add optional / configurable partitions + + result = self.collection.search( + data=[query_embedding_vector], + anns_field="embedding_vector", + param=search_params, + limit=sample_count, + output_fields=["block_mongo_id"] + ) + + else: + + search_params = { + "field_name": "embedding_vector", + "metric_type": "L2", + # "params": {"nprobe": 10} + } + + result = self.collection.search(collection_name=self.collection_name, + data=[query_embedding_vector], + anns_field="embedding_vector", + search_params=search_params, + limit=sample_count, + output_fields=["block_mongo_id"] + ) + + block_list = [] + for hits in result: + for hit in hits: + + if self.use_milvus_lite: + + try: + # alt: _id = int(hit["entity"]["block_mongo_id"]) + _id = hit["entity"]["block_mongo_id"] + except: + logger.warning(f"update: EmbeddingHandler - Milvus - search - unexpected - " + f"could not convert to number - {hit}") + _id = -1 + else: + _id = hit.entity.get('block_mongo_id') + + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + + if self.use_milvus_lite: + distance = hit["distance"] + else: + distance = hit.distance + + block_list.append((block, distance)) + + """ + try: + block = block_cursor.next() + block_list.append((block, hit.distance)) + except StopIteration: + # The cursor is empty (no blocks found) + continue + """ + + return block_list + + def delete_index(self): + + if not self.use_milvus_lite: + + collection = pymilvus.Collection(self.collection_name) + collection.release() + pymilvus.utility.drop_collection(self.collection_name) + pymilvus.connections.disconnect(self.milvus_alias) + + else: + # delete + res = self.collection.delete(collection_name=self.collection_name) + + # Synchronize and remove embedding flag from collection db + self.utils.unset_text_index() + + return 1 + + +class EmbeddingFAISS: + + """Implements the vector database FAISS. + + ``EmbeddingFAISS`` implements the interface to the ``FAISS`` vector database. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_faiss : EmbeddingFAISS + A new ``EmbeddingFAISS`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + global GLOBAL_FAISS_IMPORT + if not GLOBAL_FAISS_IMPORT: + if util.find_spec("faiss"): + + try: + global faiss + faiss = importlib.import_module("faiss") + GLOBAL_FAISS_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load faiss module.") + + else: + raise LLMWareException(message="Exception: need to import faiss to use this class.") + + # end dynamic import here + + self.library = library + self.library_name = library.library_name + self.account_name = library.account_name + self.index = None + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + self.model=model + self.model_name=model_name + self.embedding_dims=embedding_dims + + # if model passed (not None), then use model name and embedding dims + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = self.model.embedding_dims + + # embedding file name here + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="faiss", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + # will leave "-" and "_" in file path, but remove "@" and " " + model_safe_path = re.sub(r"[@\/. ]", "", self.model_name).lower() + self.embedding_file_path = os.path.join(self.library.embedding_path, model_safe_path, "embedding_file_faiss") + + def create_new_embedding(self, doc_ids=None, batch_size=100): + + """ Load or create index """ + + if not self.index: + if os.path.exists(self.embedding_file_path): + + # faiss is optional dependency + # note: there may be an edge case where this faiss command would fail even with + # library installed, but we throw dependency not installed error as most likely cause + + try: + self.index = faiss.read_index(self.embedding_file_path) + except: + raise DependencyNotInstalledException("faiss-cpu") + else: + try: + self.index = faiss.IndexFlatL2(self.embedding_dims) + except: + raise DependencyNotInstalledException("faiss-cpu") + + # get cursor for text collection with blocks requiring embedding + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.account_name) + status.new_embedding_status(self.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + finished = False + + while not finished: + + block_ids, sentences = [], [] + current_index = self.index.ntotal + + # Build the next batch + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + + text_search = block["text_search"].strip() + + if not text_search or len(text_search) < 1: + continue + block_ids.append(str(block["_id"])) + + sentences.append(text_search) + + if len(sentences) > 0: + # Process the batch + vectors = self.model.embedding(sentences) + self.index.add(np.array(vectors)) + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add options to display/hide + logger.info(f"update: embedding_handler - FAISS - Embeddings Created: {embeddings_created} of {num_of_blocks}") + + # Ensure any existing file is removed before saving + if os.path.exists(self.embedding_file_path): + os.remove(self.embedding_file_path) + os.makedirs(os.path.dirname(self.embedding_file_path), exist_ok=True) + faiss.write_index(self.index, self.embedding_file_path) + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - FAISS - embedding_summary - {embedding_summary}") + + return embedding_summary + + def search_index (self, query_embedding_vector, sample_count=10): + + """ Search FAISS index """ + + if not self.index: + self.index = faiss.read_index(self.embedding_file_path) + + distance_list, index_list = self.index.search(np.array([query_embedding_vector]), sample_count) + + block_list = [] + for i, index in enumerate(index_list[0]): + + index_int = int(index.item()) + + # FAISS is unique in that it requires a 'reverse lookup' to match the FAISS index in the + # text collection + + block_result_list = self.utils.lookup_embedding_flag(self.collection_key,index_int) + + # block_result_list = self.utils.lookup_text_index(index_int, key=self.collection_key) + + for block in block_result_list: + block_list.append((block, distance_list[0][i])) + + return block_list + + def delete_index(self): + + """ Delete FAISS index """ + + if os.path.exists(self.embedding_file_path): + os.remove(self.embedding_file_path) + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 1 + + +class EmbeddingLanceDB: + + """Implements the vector database LanceDB. + + ``EmbeddingLanceDB`` implements the interface to the ``LanceDB`` vector database. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_lancedb : EmbeddingLanceDB + A new ``EmbeddingLanceDB`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + # confirm that lancedb installed + + global GLOBAL_LANCEDB_IMPORT + if not GLOBAL_LANCEDB_IMPORT: + if util.find_spec("lancedb"): + + try: + global lancedb + lancedb = importlib.import_module("lancedb") + GLOBAL_LANCEDB_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load lancedb module.") + + else: + raise LLMWareException(message="Exception: need to import lancedb to use this class.") + + # end dynamic import here + + self.uri = LanceDBConfig().get_config("uri") + self.library = library + self.library_name = self.library.library_name + self.account_name = self.library.account_name + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + + # if model passed (not None), then use model name + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = model.embedding_dims + + # initialize LanceDB + self.index = None + + # initiate connection to LanceDB locally + try: + self.db = lancedb.connect(self.uri) + except: + raise ImportError( + "Exception - could not connect to LanceDB - please check:" + "1. LanceDB python package is installed, e.g,. 'pip install lancedb', and" + "2. The uri is properly set.") + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="lancedb", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + if self.collection_name not in self.db.table_names(): + self.index = self._init_table(self.collection_name) + + # you don't need to create an index with lanceDB up to million vectors is efficiently supported with peak performance, + # Creating an index will accelerate the search process and it needs to be done once table has some vectors already. + + # connect to table + self.index = self.db.open_table(self.collection_name) + + def _init_table(self,table_name): + + try: + import pyarrow as pa + except: + raise DependencyNotInstalledException("pyarrow") + + schema = pa.schema([ + pa.field("vector", pa.list_(pa.float32(), int(self.embedding_dims))), + pa.field("id", pa.string()), + ]) + tbl = self.db.create_table(table_name, schema=schema, mode="overwrite") + return tbl + + def create_new_embedding(self, doc_ids = None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + + # starting current_index @ 0 + current_index = 0 + + finished = False + + while not finished: + block_ids, doc_ids, sentences = [], [], [] + # Build the next batch + for i in range(batch_size): + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + # Process the batch + vectors = self.model.embedding(sentences) + + # expects records as tuples - (batch of _ids, batch of vectors, batch of dict metadata) + # records = zip(block_ids, vectors) #, doc_ids) + # upsert to lanceDB + try : + vectors_ingest = [{ 'id' : block_id,'vector': vector.tolist()} for block_id,vector in zip(block_ids,vectors)] + self.index.add(vectors_ingest) + except Exception as e : + raise LLMWareException(message=f"Exception: LanceDB - {e} - {self.index} - schema - " + f"{self.index.schema}") + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add options to configure to show/hide + logger.info (f"update: embedding_handler - Lancedb - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - Lancedb - embedding_summary - {embedding_summary}") + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + try: + result = self.index.search(query=query_embedding_vector.tolist())\ + .select(["id", "vector"])\ + .limit(sample_count).to_pandas() + + block_list = [] + + for (_, id, vec, score) in result.itertuples(name=None): + _id = id + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + block_list.append((block, score)) + + except Exception as e: + raise LLMWareException(message=f"Exception: LanceDB - {e}") + + return block_list + + def delete_index(self): + + self.db.drop_table(self.collection_name) + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 1 + + +class EmbeddingPinecone: + + """Implements the vector database Pinecone. + + ``EmbeddingPinecone`` implements the interface to the ``Pinecone`` vector database. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_pinecone : EmbeddingPinecone + A new ``EmbeddingPinecone`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + self.api_key = PineconeConfig().get_config("pinecone_api_key") + self.cloud = PineconeConfig().get_config("pinecone_cloud") + self.region = PineconeConfig().get_config("pinecone_region") + + self.library = library + self.library_name = self.library.library_name + self.account_name = self.library.account_name + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + + # if model passed (not None), then use model name + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = model.embedding_dims + + # initialize pinecone + self.index = None + + global GLOBAL_PINECONE_IMPORT + if not GLOBAL_PINECONE_IMPORT: + if util.find_spec("pinecone"): + + try: + global pinecone + pinecone = importlib.import_module("pinecone") + GLOBAL_PINECONE_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load pinecone module.") + + else: + raise LLMWareException(message="Exception: need to import pinecone to use this class.") + + # initiate connection to Pinecone + try: + pinecone_client = pinecone.Pinecone(api_key=self.api_key) + except: + raise ImportError( + "Exception - could not connect to Pinecone - please check:" + "1. Pinecone python package is installed, e.g,. 'pip install pinecone-client', and" + "2. The api key and environment is properly set.") + + # check index name - pinecone - 45 chars - numbers, letters and "-" ok - no "_" and all lowercase + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="pinecone", + embedding_dims=self.embedding_dims) + + collection_name = self.utils.create_safe_collection_name() + self.collection_name = collection_name.replace('_', '-') + + self.collection_key = self.utils.create_db_specific_key() + + pinecone_indexes = [pincone_index['name'] for pincone_index in pinecone_client.list_indexes()] + if self.collection_name not in pinecone_indexes: + pinecone_client.create_index( + name=self.collection_name, + dimension=self.embedding_dims, + metric="euclidean", + spec=pinecone.ServerlessSpec( + cloud=self.cloud, + region=self.region)) + + pinecone_client.describe_index(self.collection_name) # Waits for index to be created + # describe_index_stats() # Returns: {'dimension': 8, 'index_fullness': 0.0, 'namespaces': {'': {'vector_count': 5}}} + + # connect to index + self.index = pinecone_client.Index(self.collection_name) + + def create_new_embedding(self, doc_ids = None, batch_size=100): + + def chunks(iterable, batch_size=100): + """A helper function to break an iterable into chunks of size batch_size.""" + it = iter(iterable) + chunk = tuple(itertools.islice(it, batch_size)) + while chunk: + yield chunk + chunk = tuple(itertools.islice(it, batch_size)) + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + + # starting current_index @ 0 + current_index = 0 + + finished = False + + while not finished: + block_ids, doc_ids, sentences = [], [], [] + # Build the next batch + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + # Process the batch + vectors = self.model.embedding(sentences) + + # expects records as tuples - (batch of _ids, batch of vectors, batch of dict metadata) + records = zip(block_ids, vectors) #, doc_ids) + # upsert to Pinecone + + # Upsert data with 100 vectors per upsert request + for records_chunk in chunks(records, batch_size=100): + self.index.upsert(vectors=records_chunk) + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add options to configure to show/hide + logger.info (f"update: embedding_handler - Pinecone - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - Pinecone - embedding_summary - {embedding_summary}") + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + + result = self.index.query(vector=query_embedding_vector, top_k=sample_count,include_values=True) + + block_list = [] + for match in result["matches"]: + _id = match["id"] + + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + block_list.append((block, match["score"])) + + return block_list + + def delete_index(self, index_name): + + pinecone.delete_index(index_name) + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 1 + + +class EmbeddingMongoAtlas: + """Implements the use of MongoDB Atlas as a vector database. + + ``EmbeddingMongoAtlas`` implements the interface to ``MongoDB Atlas``. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_mongoatlas : EmbeddingMongoAtlas + A new ``EmbeddingMongoAtlas`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + # Use a specified Mongo Atlas connection string if supplied. + # Otherwise fallback to the the Mongo DB connection string + # self.connection_uri = os.environ.get("MONGO_ATLAS_CONNECTION_URI", MongoConfig.get_config("collection_db_uri")) + + self.connection_uri = MongoConfig().get_config("atlas_db_uri") + + self.library = library + self.library_name = self.library.library_name + self.account_name = self.library.account_name + + # look up model card + self.model_name = model.model_name + self.model = model + self.embedding_dims = embedding_dims + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + # if model passed (not None), then use model name + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = model.embedding_dims + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="mongoatlas", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + # Connect and create a MongoClient + # confirm that pymongo installed + + global GLOBAL_PYMONGO_IMPORT + if not GLOBAL_PYMONGO_IMPORT: + if util.find_spec("pymongo"): + + try: + global pymongo + pymongo = importlib.import_module("pymongo") + GLOBAL_PYMONGO_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load pymongo module.") + + else: + raise LLMWareException(message="Exception: need to import pymongo to use this class.") + + # end dynamic import here + # alt: from pymongo import MongoClient + + self.mongo_client = pymongo.MongoClient(self.connection_uri) + + # Make sure the Database exists by creating a dummy metadata collection + self.embedding_db_name = "llmware_embeddings" + self.embedding_db = self.mongo_client["llmware_embeddings"] + if self.embedding_db_name not in self.mongo_client.list_database_names(): + self.embedding_db["metadata"].insert_one({"created": Utilities().get_current_time_now()}) + + # Connect to collection and create it if it doesn't exist by creating a dummy doc + self.embedding_collection = self.embedding_db[self.collection_name] + if self.collection_name not in self.embedding_db.list_collection_names(): + self.embedding_collection.insert_one({"created": Utilities().get_current_time_now()}) + + # If the collection does not have a search index (e.g if it's new), create one + if len (list(self.embedding_collection.list_search_indexes())) < 1: + model = { + 'name': self.collection_name, + 'definition': { + 'mappings': { + 'dynamic': True, + 'fields': { + 'eVector': { + 'type': 'knnVector', + 'dimensions': self.embedding_dims, + 'similarity': 'euclidean' + }, + } + } + } + } + + self.embedding_collection.create_search_index(model) + + def create_new_embedding(self, doc_ids = None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + + # starting current_index @ 0 + current_index = 0 + + finished = False + + last_block_id = "" + + while not finished: + + block_ids, doc_ids, sentences = [], [], [] + # Build the next batch + + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + # Process the batch + vectors = self.model.embedding(sentences).tolist() + + docs_to_insert = [] + for i, vector in enumerate(vectors): + doc = { + "id": str(block_ids[i]), + "doc_ID": str(doc_ids[i]), + "eVector": vector + } + docs_to_insert.append(doc) + + insert_result = self.embedding_collection.insert_many(docs_to_insert) + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add configuration options to hide/show + logger.info(f"update: embedding_handler - Mongo Atlas - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + last_block_id = block_ids[-1] + + if embeddings_created > 0: + + logger.info(f"Embedding(Mongo Atlas): Waiting for {self.embedding_db_name}.{self.collection_name} " + f"to be ready for vector search...") + + start_time = time.time() + self.wait_for_search_index(last_block_id, start_time) + wait_time = time.time() - start_time + + logger.info(f"Embedding(Mongo Atlas): {self.embedding_db_name}.{self.collection_name} " + f"ready ({wait_time: .2f} seconds)") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - Mongo Atlas - embedding_summary - {embedding_summary}") + + return embedding_summary + + def wait_for_search_index(self, last_block_id, start_time): + + """ After doc insertion, we want to make sure the index is ready before proceeding ... """ + + # if wait longer than 5 mins, then time out and just return + if time.time() > start_time + (5 * 60): + return + + # Get the atlas search index + the_index = self.embedding_collection.list_search_indexes().next() + + # If the index doesn't have status="READY" or queryable=True, wait + if the_index["status"] != "READY" or not the_index["queryable"]: + time.sleep(3) + return self.wait_for_search_index(last_block_id, start_time) + + # If we can't find the last block yet in the search index, wait + search_query = { + "$search": { + "index": self.collection_name, + "text": { + "query": str(last_block_id), + "path": "id" # The field in your documents you're matching against + } + } + } + results = self.embedding_collection.aggregate([search_query]) + if not results.alive: + time.sleep(1) + return self.wait_for_search_index(last_block_id, start_time) + + def search_index(self, query_embedding_vector, sample_count=10): + + search_results = self.embedding_collection.aggregate([ + { + "$vectorSearch": { + "index": self.collection_name, + "path": "eVector", + "queryVector": query_embedding_vector.tolist(), + "numCandidates": sample_count * 10, # Following recommendation here: https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/ + "limit": sample_count + } + }, + { + "$project": { + "_id": 0, + "id": 1, + "doc_ID": 1, + "score": { "$meta": "vectorSearchScore" } + } + } + ]) + + block_list = [] + for search_result in search_results: + _id = search_result["id"] + + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + distance = 1 - search_result["score"] # Atlas returns a score from 0 to 1.0 + block_list.append((block, distance)) + + return block_list + + def delete_index(self, index_name): + + self.embedding_db.drop_collection(index_name) + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 1 + + +class EmbeddingRedis: + + """Implements the use of Redis as a vector database. + + ``EmbeddingRedis`` implements the interface to ``Redis``. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_redis : EmbeddingRedis + A new ``EmbeddingRedis`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + self.library = library + self.library_name = library.library_name + self.account_name = library.account_name + + # Connect to redis - use "localhost" & 6379 by default + redis_host = RedisConfig().get_config("host") + redis_port = RedisConfig().get_config("port") + + # confirm that redis installed + + global GLOBAL_REDIS_IMPORT + if not GLOBAL_REDIS_IMPORT: + if util.find_spec("redis"): + + try: + global redis + redis= importlib.import_module("redis") + GLOBAL_REDIS_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load redis module.") + + else: + raise LLMWareException(message="Exception: need to import redis to use this class.") + + # end dynamic import here + + self.r = redis.Redis(host=redis_host, port=redis_port, decode_responses=True) + + # look up model card + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = self.model.embedding_dims + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="redis", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + self.DOC_PREFIX = self.collection_name # key prefix used for the index + + try: + # check to see if index exists + self.r.ft(self.collection_name).info() + logger.info("update: embedding_handler - Redis - index already exists - %s", self.collection_name) + + except: + + from redis.commands.search import field + # schema + schema = ( + field.NumericField("id"), + field.TextField("text"), + field.TextField("block_mongo_id"), + field.NumericField("block_id"), + field.NumericField("block_doc_id"), + field.VectorField("vector", # Vector Field Name + "FLAT", { # Vector Index Type: FLAT or HNSW + "TYPE": "FLOAT32", # FLOAT32 or FLOAT64 + "DIM": self.embedding_dims, + "DISTANCE_METRIC": "L2", # "COSINE" alternative + } + ), + ) + + # index Definition + from redis.commands.search.indexDefinition import IndexDefinition, IndexType + definition = IndexDefinition(prefix=[self.DOC_PREFIX], + index_type=IndexType.HASH) + + # create Index + self.r.ft(self.collection_name).create_index(fields=schema, definition=definition) + + logger.info("update: embedding_handler - Redis - creating new index - %s ", self.collection_name) + + def create_new_embedding(self, doc_ids=None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + current_index = 0 + finished = False + + obj_batch = [] + + while not finished: + + block_ids, doc_ids, sentences = [], [], [] + + # Build the next batch + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + obj = {"block_mongo_id": str(block["_id"]), + "block_doc_id": int(block["doc_ID"]), + "block_id": int(block["block_ID"]), + "text": text_search + } + + obj_batch.append(obj) + + if len(sentences) > 0: + + # Process the batch + vectors = self.model.embedding(sentences) + + pipe = self.r.pipeline() + + for i, embedding in enumerate(vectors): + + redis_dict = obj_batch[i] + + embedding = np.array(embedding) + redis_dict.update({"vector": embedding.astype(np.float32).tobytes()}) + key_name = f"{self.DOC_PREFIX}:{redis_dict['block_mongo_id']}" + + pipe.hset(key_name, mapping=redis_dict) + + res = pipe.execute() + obj_batch = [] + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add configuration options to show/display + logger.info(f"update: embedding_handler - Redis - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - Redis - embedding_summary - {embedding_summary}") + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + + query_embedding_vector = np.array(query_embedding_vector) + + query = ( + redis.commands.search.query.Query(f"*=>[KNN {sample_count} @vector $vec as score]") + .sort_by("score") + .return_fields("score", "block_mongo_id", "block_doc_id", "block_id","text") + .paging(0, sample_count) + .dialect(2) + ) + + query_params = { + "vec": query_embedding_vector.astype(np.float32).tobytes() + } + + results = self.r.ft(self.collection_name).search(query, query_params).docs + + block_list = [] + for j, res in enumerate(results): + + _id = str(res["block_mongo_id"]) + score = float(res["score"]) + + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + block_list.append((block, score)) + + return block_list + + def delete_index(self): + + # delete index + self.r.ft(self.collection_name).dropindex(delete_documents=True) + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 0 + + +class EmbeddingQdrant: + + """Implements the Qdrant vector database. + + ``EmbeddingQdrant`` implements the interface to ``Qdrant``. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_qdrant : EmbeddingQdrant + A new ``EmbeddingQdrant`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + self.library = library + self.library_name = library.library_name + self.account_name = library.account_name + + # confirm that qdrant installed + + global GLOBAL_QDRANT_IMPORT + if not GLOBAL_QDRANT_IMPORT: + if util.find_spec("qdrant_client"): + + try: + global qdrant_client + qdrant_client = importlib.import_module("qdrant_client") + GLOBAL_QDRANT_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load qdrant_client module.") + + else: + raise LLMWareException(message="Exception: need to import qdrant_client to use this class.") + + # end dynamic import here + + self.qclient = qdrant_client.QdrantClient(**QdrantConfig.get_config()) + + # look up model card + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = self.model.embedding_dims + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="qdrant", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + # check if collection already exists, or if needs to be created + + collections = self.qclient.get_collections() + collection_exists = False + + for i, cols in enumerate(collections.collections): + if cols.name == self.collection_name: + collection_exists = True + break + + if not collection_exists: + + self.collection = ( + self.qclient.create_collection( + collection_name=self.collection_name, + vectors_config=qdrant_client.http.models.VectorParams(size=self.embedding_dims, + distance=qdrant_client.http.models.Distance.DOT), )) + + logger.info("update: embedding_handler - QDRANT - creating new collection - %s", + self.collection_name) + + else: + # if collection already exists, then 'get' collection + self.collection = self.qclient.get_collection(self.collection_name) + + def create_new_embedding(self, doc_ids=None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + current_index = 0 + finished = False + + points_batch = [] + + while not finished: + + block_ids, doc_ids, sentences = [], [], [] + + # Build the next batch + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + + # Process the batch + vectors = self.model.embedding(sentences) + + for i, embedding in enumerate(vectors): + + point_id = str(uuid.uuid4()) + ps = qdrant_client.http.models.PointStruct(id=point_id, vector=embedding, + payload={"block_doc_id": doc_ids[i], + "sentences": sentences[i], + "block_mongo_id": block_ids[i]}) + + points_batch.append(ps) + + # upsert a batch of points + self.qclient.upsert(collection_name=self.collection_name, wait=True, points=points_batch) + + points_batch = [] + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add configuration options to show/display + logger.info(f"update: embedding_handler - Qdrant - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + + logger.info(f"update: EmbeddingHandler - Qdrant - embedding_summary - {embedding_summary}") + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + + search_results = self.qclient.search(collection_name=self.collection_name, + query_vector=query_embedding_vector, limit=sample_count) + + block_list = [] + for j, res in enumerate(search_results): + + _id = res.payload["block_mongo_id"] + + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + block_list.append((block, res.score)) + + return block_list + + def delete_index(self): + + # delete index - need to add + self.qclient.delete_collection(collection_name=f"{self.collection_name}") + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 0 + + +class EmbeddingPGVector: + + """Implements the interface to the PGVector vector database. + + ``EmbeddingPGVector`` implements the interface to ``PGVector``. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_pgvector : EmbeddingPGVector + A new ``EmbeddingPGVector`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None, full_schema=False): + + self.library = library + self.library_name = library.library_name + self.account_name = library.account_name + + # look up model card + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = self.model.embedding_dims + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="pg_vector", + embedding_dims=self.embedding_dims) + + self.collection_name = self.utils.create_safe_collection_name() + self.collection_key = self.utils.create_db_specific_key() + + # Connect to postgres + postgres_host = PostgresConfig().get_config("host") + postgres_port = PostgresConfig().get_config("port") + postgres_db_name = PostgresConfig().get_config("db_name") + postgres_user_name = PostgresConfig().get_config("user_name") + postgres_pw = PostgresConfig().get_config("pw") + postgres_schema = PostgresConfig().get_config("pgvector_schema") + + # default schema captures only minimum required for tracking vectors + if postgres_schema == "vector_only": + self.full_schema = False + else: + self.full_schema = True + + # determines whether to use 'skinny' schema or 'full' schema + # --note: in future releases, we will be building out more support for Postgres + + # first check for core postgres driver, and load if not present + global GLOBAL_PSYCOPG_IMPORT + if not GLOBAL_PSYCOPG_IMPORT: + if util.find_spec("psycopg"): + + try: + global psycopg + psycopg = importlib.import_module("psycopg") + GLOBAL_PSYCOPG_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load psycopg module.") + + else: + raise LLMWareException(message="Exception: need to import psycopg to use this class.") + + # second check for pg_vector specific driver and load if not present + global GLOBAL_PGVECTOR_IMPORT + if not GLOBAL_PGVECTOR_IMPORT: + if util.find_spec("pgvector"): + + try: + global pgvector + pgvector = importlib.import_module("pgvector") + GLOBAL_PGVECTOR_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load pgvector module.") + + else: + raise LLMWareException(message="Exception: need to import neo4j to use this class.") + + # note: for initial connection, need to confirm that the database exists + self.conn = psycopg.connect(host=postgres_host, port=postgres_port, dbname=postgres_db_name, + user=postgres_user_name, password=postgres_pw) + + # register vector extension + self.conn.execute('CREATE EXTENSION IF NOT EXISTS vector') + from pgvector.psycopg import register_vector + register_vector(self.conn) + + if not self.full_schema: + + table_create = (f"CREATE TABLE IF NOT EXISTS {self.collection_name} " + f"(id bigserial PRIMARY KEY, " + f"text text, " + f"embedding vector({self.embedding_dims}), " + f"block_mongo_id text, " + f"block_doc_id integer);") + else: + + # full schema is a replica of the Mongo parsing output key structure + + table_create = (f"CREATE TABLE IF NOT EXISTS {self.collection_name} " + f"(id bigserial PRIMARY KEY, " + f"embedding vector({self.embedding_dims})," + f"block_mongo_id text, " + f"block_doc_id integer," + f"block_ID integer, " + f"doc_ID integer, " + f"content_type text, " + f"file_type text, " + f"master_index integer, " + f"master_index2 integer, " + f"coords_x integer, " + f"coords_y integer, " + f"coords_cx integer, " + f"coords_cy integer, " + f"author_or_speaker text, " + f"modified_date text, " + f"created_date text, " + f"creator_tool text," + f"added_to_collection text," + f"table_block text," + f"text text," + f"external_files text," + f"file_source text," + f"header_text text," + f"text_search text," + f"user_tags text," + f"special_field1 text," + f"special_field2 text," + f"special_field3 text," + f"graph_status text," + f"embedding_flags json," + f"dialog text);") + + # execute the creation of the table, if needed + self.conn.execute(table_create) + self.conn.commit() + + def create_new_embedding(self, doc_ids=None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + current_index = 0 + finished = False + + obj_batch = [] + + while not finished: + + block_ids, doc_ids, sentences = [], [], [] + + # Build the next batch + for i in range(batch_size): + + block = all_blocks_cursor.pull_one() + + if not block: + finished = True + break + + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if not self.full_schema: + + obj = {"block_mongo_id": str(block["_id"]), + "block_doc_id": int(block["doc_ID"]), + "text": text_search} + else: + obj = {} + for keys in block: + if keys == "_id": + value = str(block["_id"]) + obj.update({"block_mongo_id": value}) + else: + value = block[keys] + obj.update({keys:value}) + obj.update({"block_doc_id": int(block["doc_ID"])}) + + obj_batch.append(obj) + + if len(sentences) > 0: + + # Process the batch + vectors = self.model.embedding(sentences) + + for i, embedding in enumerate(vectors): + + if not self.full_schema: + + insert_command=(f"INSERT INTO {self.collection_name} (text, embedding, block_mongo_id," + f"block_doc_id) VALUES (%s, %s, %s, %s)") + + insert_array=(obj_batch[i]["text"], embedding, + obj_batch[i]["block_mongo_id"], obj_batch[i]["block_doc_id"],) + + else: + + insert_command=(f"INSERT INTO {self.collection_name} " + f"(embedding, block_mongo_id, block_doc_id," + f"block_ID, doc_ID, content_type, file_type, master_index," + f"master_index2, coords_x, coords_y,coords_cx, coords_cy," + f"author_or_speaker, modified_date, created_date, creator_tool," + f"added_to_collection, table_block, text, external_files,file_source," + f"header_text, text_search, user_tags, special_field1, special_field2," + f"special_field3, graph_status, dialog) " + f"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " + f"%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " + f"%s, %s, %s, %s)") + + insert_array=(embedding, obj_batch[i]["block_mongo_id"], + obj_batch[i]["block_doc_id"], obj_batch[i]["block_ID"], + obj_batch[i]["doc_ID"], obj_batch[i]["content_type"], + obj_batch[i]["file_type"], obj_batch[i]["master_index"], + obj_batch[i]["master_index2"], obj_batch[i]["coords_x"], + obj_batch[i]["coords_y"], obj_batch[i]["coords_cx"], + obj_batch[i]["coords_cy"], obj_batch[i]["author_or_speaker"], + obj_batch[i]["modified_date"], obj_batch[i]["created_date"], + obj_batch[i]["creator_tool"], obj_batch[i]["added_to_collection"], + obj_batch[i]["table"], obj_batch[i]["text"], obj_batch[i]["external_files"], + obj_batch[i]["file_source"], obj_batch[i]["header_text"], + obj_batch[i]["text_search"], obj_batch[i]["user_tags"], + obj_batch[i]["special_field1"], obj_batch[i]["special_field2"], obj_batch[i]["special_field3"], + obj_batch[i]["graph_status"], obj_batch[i]["dialog"]) + + self.conn.execute(insert_command, insert_array) + + self.conn.commit() + + obj_batch = [] + + current_index = self.utils.update_text_index(block_ids,current_index) + + embeddings_created += len(sentences) + + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + # will add configuration options to show/display + logger.info(f"update: embedding_handler - PGVector - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + embedded_blocks = embedding_summary["embedded_blocks"] + + logger.info(f"update: EmbeddingHandler - PG_Vector - embedding_summary - {embedding_summary}") + + # safety check on output + if not isinstance(embedded_blocks, int): + if len(embedded_blocks) > 0: + embedded_blocks = embedded_blocks[0] + else: + embedded_blocks = embeddings_created + + # create index + lists = max(embedded_blocks // 1000, 10) + + create_index_command = (f"CREATE INDEX ON {self.collection_name} " + f"USING ivfflat(embedding vector_l2_ops) WITH(lists={lists});") + + self.conn.execute(create_index_command) + self.conn.commit() + + # TODO - add options to create text index and options to query directly against PG + + # Closing the connection + self.conn.close() + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + + # note: converting to np.array is 'safety' for postgres vector type + query_embedding_vector = np.array(query_embedding_vector) + + q = (f"SELECT id, block_mongo_id, embedding <-> %s AS distance, text " + f"FROM {self.collection_name} ORDER BY distance LIMIT %s") + + """ + # alt - look to generalize the query + q = (f"SELECT embedding <-> %s AS distance, * FROM {self.collection_name} ORDER BY " + f"distance LIMIT %s") + """ + + cursor = self.conn.cursor() + results = cursor.execute(q, (query_embedding_vector,sample_count)) + + block_list = [] + for j, res in enumerate(results): + + pg_id = res[0] + _id = res[1] + distance = res[2] + text = res[3] + + block_result_list = self.utils.lookup_text_index(_id) + + for block in block_result_list: + block_list.append((block, distance)) + + # Closing the connection + self.conn.close() + + return block_list + + def delete_index(self, collection_name=None): + + # delete index - drop table + if collection_name: + self.collection_name = collection_name + + drop_command = f'''DROP TABLE {self.collection_name} ''' + + # Executing the query + cursor = self.conn.cursor() + cursor.execute(drop_command) + + logger.info("update: embedding_handler - PG Vector - table dropped - %s", self.collection_name) + + # Commit your changes in the database + self.conn.commit() + + # Closing the connection + self.conn.close() + + # remove emb key - 'unset' the blocks in the text collection + self.utils.unset_text_index() + + return 0 + + +class EmbeddingNeo4j: + + """Implements the interface to Neo4j as a vector database. + + ``EmbeddingNeo4j`` implements the interface to ``Neo4j``. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_Neo4j : EmbeddingNeo4j + A new ``EmbeddingNeo4j`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + self.library = library + self.library_name = library.library_name + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + self.account_name = library.account_name + + # if model passed (not None), then use model name + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = model.embedding_dims + + # user and password names are taken from environmen variables + # Names for user and password are taken from the link below + # https://neo4j.com/docs/operations-manual/current/tools/neo4j-admin/upload-to-aura/#_options + uri = Neo4jConfig.get_config('uri') + user = Neo4jConfig.get_config('user') + password = Neo4jConfig.get_config('password') + database = Neo4jConfig.get_config('database') + + global GLOBAL_NEO4J_IMPORT + if not GLOBAL_NEO4J_IMPORT: + if util.find_spec("neo4j"): + + try: + global neo4j + neo4j = importlib.import_module("neo4j") + GLOBAL_NEO4J_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load neo4j module.") + + else: + raise LLMWareException(message="Exception: need to import neo4j to use this class.") + + # end dynamic import here + + # Connect to Neo4J and verify connection. + # Code taken from the code below + # https://github.com/langchain-ai/langchain/blob/master/libs/community/langchain_community/vectorstores/neo4j_vector.py#L165C9-L177C14 + try: + self.driver = neo4j.GraphDatabase.driver(uri, auth=(user, password)) + self.driver.verify_connectivity() + except neo4j.exceptions.ServiceUnavailable: + raise ValueError( + "Could not connect to Neo4j database. " + "Please ensure that the url is correct and that Neo4j is up and running.") + except neo4j.exceptions.AuthError: + raise ValueError( + "Could not connect to Neo4j database. " + "Please ensure that the username and password are correct.") + except Exception as err: + # We raise here any other exception that happened. + # This is useful for debugging when some other error occurs. + raise + + # Make sure that the Neo4j version supports vector indexing. + neo4j_version = self._query('call dbms.components() ' + 'yield name, versions, edition ' + 'unwind versions as version ' + 'return version')[0]['version'] + + neo4j_version = tuple(map(int, neo4j_version.split('.'))) + + target_version = (5, 11, 0) + if neo4j_version < target_version: + raise ValueError('Vector indexing requires a Neo4j version >= 5.11.0') + + # If the index does not exist, then we create the vector search index. + neo4j_indexes = self._query('SHOW INDEXES yield name') + neo4j_indexes = [neo4j_index['name'] for neo4j_index in neo4j_indexes] + if 'vectorIndex' not in neo4j_indexes: + self._query( + query='CALL ' + 'db.index.vector.createNodeIndex(' + '$indexName, ' + '$label, ' + '$propertyKey, ' + 'toInteger($vectorDimension), ' + '"euclidean"' + ')', + parameters={ + 'indexName': 'vectorIndex', + 'label': 'Chunk', + 'propertyKey': 'embedding', + 'vectorDimension': int(self.model.embedding_dims) + }) + + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="neo4j", + embedding_dims=self.embedding_dims) + + def create_new_embedding(self, doc_ids=None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + current_index = 0 + finished = False + + while not finished: + block_ids, doc_ids, sentences = [], [], [] + + # Build the next batch + for i in range(batch_size): + block = all_blocks_cursor.pull_one() + if not block: + finished = True + break + + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + # Process the batch + vectors = self.model.embedding(sentences) + data = [block_ids, doc_ids, vectors] + + # Insert into Neo4J + insert_query = ( + "UNWIND $data AS row " + "CALL " + "{ " + "WITH row " + "MERGE (c:Chunk {id: row.doc_id, block_id: row.block_id}) " + "WITH c, row " + "CALL db.create.setVectorProperty(c, 'embedding', row.embedding) " + "YIELD node " + "SET c.sentence = row.sentence " + "} " + f"IN TRANSACTIONS OF {batch_size} ROWS" + ) + + parameters = { + "data": [ + {"block_id": block_id, "doc_id": doc_id, "sentence": sentences, "embedding": vector} + for block_id, doc_id, sentence, vector in zip( + block_ids, doc_ids, sentences, vectors + ) + ] + } + + self._query(query=insert_query, parameters=parameters) + + current_index = self.utils.update_text_index(block_ids, current_index) + + # Update statistics + embeddings_created += len(sentences) + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + logger.info(f"update: embedding_handler - Neo4j - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + logger.info(f'update: EmbeddingHandler - Neo4j - embedding_summary - {embedding_summary}') + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + block_list = [] + + search_query = 'CALL db.index.vector.queryNodes("vectorIndex" , $sample_count, $query_embedding_vector) '\ + 'YIELD node, score ' + + parameters = {'sample_count': sample_count, 'query_embedding_vector': query_embedding_vector} + + results = self._query(query=search_query, parameters=parameters) + + for result in results: + block_id = result['node']['block_id'] + block_result_list = self.utils.lookup_text_index(block_id) + + for block in block_result_list: + block_list.append((block, result["score"])) + + return block_list + + def delete_index(self, index_name): + + try: + self._query(f"DROP INDEX $index_name", {'index_name': index_name}) + except neo4j.DatabaseError: # Index did not exist yet + pass + + self.utils.unset_text_index() + + def _query(self, query, parameters=None): + + # alt: from neo4j.exceptions import CypherSyntaxError + + parameters = parameters or {} + + with self.driver.session(database='neo4j') as session: + try: + data = session.run(query, parameters) + return [d.data() for d in data] + except neo4j.exceptions.CypherSyntaxError as e: + raise ValueError(f'Cypher Statement is not valid\n{e}') + + +class EmbeddingChromaDB: + + """Implements the interface to the ChromaDB vector database. + + ``EmbeddingChromaDB`` implements the interface to ``ChromaDB``. It is used by the + ``EmbeddingHandler``. + + Parameters + ---------- + library : object + A ``Library`` object. + + model : object + A model object. See :mod:`models` for available models. + + model_name : str, default=None + Name of the model. + + embedding_dims : int, default=None + Dimension of the embedding. + + Returns + ------- + embedding_chromadb : EmbeddingChromaDB + A new ``EmbeddingPGVector`` object. + """ + + def __init__(self, library, model=None, model_name=None, embedding_dims=None): + + # + # General llmware set up code + # + # confirm that pymilvus installed + + global GLOBAL_CHROMADB_IMPORT + if not GLOBAL_CHROMADB_IMPORT: + if util.find_spec("chromadb"): + + try: + global chromadb + chromadb = importlib.import_module("chromadb") + GLOBAL_CHROMADB_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load chromadb module.") + + else: + raise LLMWareException(message="Exception: need to import chromadb to use this class.") + + # end dynamic import here + + # look up model card + if not model and not model_name: + raise ModelNotFoundException("no-model-or-model-name-provided") + + self.library = library + self.library_name = library.library_name + self.model = model + self.model_name = model_name + self.embedding_dims = embedding_dims + self.account_name = library.account_name + + # if model passed (not None), then use model name + if self.model: + self.model_name = self.model.model_name + self.embedding_dims = model.embedding_dims + + # + # ChromaDB instantiation + # + + # Get environment variables to decide which client to use. + persistent_path = ChromaDBConfig.get_config('persistent_path') + host = ChromaDBConfig.get_config('host') + + # Instantiate client. + if not util.find_spec("chromadb"): + raise DependencyNotInstalledException("pip3 install chromadb") + + if host is None and persistent_path is None: + self.client = chromadb.EphemeralClient() + + if persistent_path is not None: + self.client = chromadb.PersistentClient(path=persistent_path) + + if host is not None: + self.client = chromadb.HttpClient(host=host, + port=ChromaDBConfig.get_config('port'), + ssl=ChromaDBConfig.get_config('ssl'), + headers=ChromaDBConfig.get_config('headers')) + + # ChromaDB Collection maps to the LLMWare library + collection_name = self.library_name + + # If the collection already exists, it is returned. + self._collection = self.client.create_collection(name=collection_name, get_or_create=True) + + # + # Embedding utils + # + + self.utils = _EmbeddingUtils(library_name=self.library_name, + model_name=self.model_name, + account_name=self.account_name, + db_name="chromadb", + embedding_dims=self.embedding_dims) + + def create_new_embedding(self, doc_ids=None, batch_size=500): + + all_blocks_cursor, num_of_blocks = self.utils.get_blocks_cursor(doc_ids=doc_ids) + + # Initialize a new status + status = Status(self.library.account_name) + status.new_embedding_status(self.library.library_name, self.model_name, num_of_blocks) + + embeddings_created = 0 + current_index = 0 + finished = False + + while not finished: + block_ids, doc_ids, sentences = [], [], [] + + # Build the next batch + for i in range(batch_size): + block = all_blocks_cursor.pull_one() + if not block: + finished = True + break + + text_search = block["text_search"].strip() + if not text_search or len(text_search) < 1: + continue + + block_ids.append(str(block["_id"])) + doc_ids.append(int(block["doc_ID"])) + sentences.append(text_search) + + if len(sentences) > 0: + # Process the batch + vectors = self.model.embedding(sentences) + + # Insert into ChromaDB + ids = [f'{doc_id}-{block_id}' for doc_id, block_id in zip(doc_ids, block_ids)] + metadatas = [{'doc_id': doc_id, 'block_id': block_id, 'sentence': sentence} + for doc_id, block_id, sentence in zip(doc_ids, block_ids, sentences)] + + self._collection.add(ids=ids, + # documents=doc_ids, + embeddings=vectors, + metadatas=metadatas) + + current_index = self.utils.update_text_index(block_ids, current_index) + + # Update statistics + embeddings_created += len(sentences) + status.increment_embedding_status(self.library.library_name, self.model_name, len(sentences)) + + logger.info(f"update: embedding_handler - ChromaDB - Embeddings Created: " + f"{embeddings_created} of {num_of_blocks}") + + embedding_summary = self.utils.generate_embedding_summary(embeddings_created) + logger.info(f'update: EmbeddingHandler - ChromaDB - embedding_summary - {embedding_summary}') + + return embedding_summary + + def search_index(self, query_embedding_vector, sample_count=10): + + block_list = [] + + # add one dimension because chroma expects two dimensions - a list of lists + query_embedding_vector = query_embedding_vector.reshape(1, -1) + + results = self._collection.query(query_embeddings=query_embedding_vector, n_results=sample_count) + + for idx_result, _ in enumerate(results['ids'][0]): + block_id = results['metadatas'][0][idx_result]['block_id'] + block_result_list = self.utils.lookup_text_index(block_id) + + for block in block_result_list: + block_list.append((block, results['distances'][0][idx_result])) + + return block_list + + def delete_index(self): + + self.client.delete_collection(self._collection.name) + self.utils.unset_text_index() diff --git a/llmware/gguf_configs.py b/llmware/gguf_configs.py new file mode 100644 index 0000000..6f06b93 --- /dev/null +++ b/llmware/gguf_configs.py @@ -0,0 +1,1362 @@ + +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +""" GGUF Configs module implements the 'internal' ctypes interfaces into llama cpp, which is referenced +in llmware in the GGUFGenerativeModel class in the models module. For more information, +see: + + -- Llama CPP: www.github.com/ggerganov/llama.cpp + -- Python binding: www.github.com/abetlen/llama_cpp_python + +""" + +import logging +import os +import numpy as np +import sys +import time +import multiprocessing +from typing import NewType +from llmware.configs import LLMWareException, ModelNotFoundException + +logger = logging.getLogger(__name__) +import ctypes +from dataclasses import field + + +LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1 +LLAMA_ROPE_SCALING_TYPE_NONE = 0 +LLAMA_ROPE_SCALING_TYPE_LINEAR = 1 +LLAMA_ROPE_SCALING_TYPE_YARN = 2 +LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3 +LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_YARN + +LLAMA_POOLING_TYPE_UNSPECIFIED = -1 +LLAMA_POOLING_TYPE_NONE = 0 +LLAMA_POOLING_TYPE_MEAN = 1 +LLAMA_POOLING_TYPE_CLS = 2 +LLAMA_POOLING_TYPE_LAST = 3 +LLAMA_POOLING_TYPE_RANK = 4 + +LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1 +LLAMA_ATTENTION_TYPE_CAUSAL = 0 +LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1 + +LLAMA_SPLIT_MODE_NONE = 0 +LLAMA_SPLIT_MODE_LAYER = 1 +LLAMA_SPLIT_MODE_ROW = 2 + + +# Ctypes Struct wrappers that map to llama.cpp C++/C objects +llama_model_p_ctypes = ctypes.c_void_p +llama_context_p_ctypes = ctypes.c_void_p +llama_pos = ctypes.c_int32 +llama_token = ctypes.c_int32 +llama_token_p = ctypes.POINTER(llama_token) +llama_seq_id = ctypes.c_int32 +ggml_backend_sched_eval_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p) +llama_log_callback = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p) +llama_grammar_p = ctypes.c_void_p +llama_progress_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_float, ctypes.c_void_p) + +llama_memory_t_ctypes = ctypes.c_void_p +llama_vocab_p_ctypes = ctypes.c_void_p + +# whisper_log_callback mirrors llama_log_callback +whisper_log_callback = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p) + + +class llama_token_data(ctypes.Structure): + + _fields_ = [ + ("id", llama_token), + ("logit", ctypes.c_float), + ("p", ctypes.c_float), + ] + +llama_token_data_p = ctypes.POINTER(llama_token_data) + +class llama_token_data_array(ctypes.Structure): + + _fields_ = [ + ("data", llama_token_data_p), + ("size", ctypes.c_size_t), + ("sorted", ctypes.c_bool), + #TODO: new + ("selected", ctypes.c_int64) + ] + +llama_token_data_array_p = ctypes.POINTER(llama_token_data_array) + + +class llama_batch(ctypes.Structure): + + _fields_ = [ + ("n_tokens", ctypes.c_int32), + ("token", ctypes.POINTER(llama_token)), + ("embd", ctypes.POINTER(ctypes.c_float)), + ("pos", ctypes.POINTER(llama_pos)), + ("n_seq_id", ctypes.POINTER(ctypes.c_int32)), + ("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))), + ("logits", ctypes.POINTER(ctypes.c_int8)), + + ] + + +class llama_model_kv_override_value(ctypes.Union): + + _fields_ = [ + ("val_i64", ctypes.c_int64), + ("val_f64", ctypes.c_double), + ("val_bool", ctypes.c_bool), + ("val_str", ctypes.c_char * 128), + ] + + +class llama_model_kv_override(ctypes.Structure): + + _fields_ = [ + ("key", ctypes.c_char * 128), + ("tag", ctypes.c_int), + ("value", llama_model_kv_override_value), + ] + + +class llama_model_params(ctypes.Structure): + + _fields_ = [ + + ("devices", ctypes.c_void_p), + ("tensor_buft_overrides", ctypes.c_void_p), + ("n_gpu_layers", ctypes.c_int32), + ("split_mode", ctypes.c_int), + ("main_gpu", ctypes.c_int32), + ("tensor_split", ctypes.POINTER(ctypes.c_float)), + ("progress_callback", llama_progress_callback), + ("progress_callback_user_data", ctypes.c_void_p), + ("kv_overrides", ctypes.POINTER(llama_model_kv_override)), + ("vocab_only", ctypes.c_bool), + ("use_mmap", ctypes.c_bool), + ("use_mlock", ctypes.c_bool), + ("check_tensors", ctypes.c_bool) + ] + + +ggml_abort_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p) + + +class llama_context_params(ctypes.Structure): + + _fields_ = [ + + ("n_ctx", ctypes.c_uint32), + ("n_batch", ctypes.c_uint32), + ("n_ubatch", ctypes.c_uint32), + ("n_seq_max", ctypes.c_uint32), + ("n_threads", ctypes.c_uint32), + ("n_threads_batch", ctypes.c_uint32), + ("rope_scaling_type", ctypes.c_int), + ("pooling_type", ctypes.c_int), + ("attention_type", ctypes.c_int), + ("rope_freq_base", ctypes.c_float), + ("rope_freq_scale", ctypes.c_float), + ("yarn_ext_factor", ctypes.c_float), + ("yarn_attn_factor", ctypes.c_float), + ("yarn_beta_fast", ctypes.c_float), + ("yarn_beta_slow", ctypes.c_float), + ("yarn_orig_ctx", ctypes.c_uint32), + ("defrag_thold", ctypes.c_float), + ("cb_eval", ggml_backend_sched_eval_callback), + ("cb_eval_user_data", ctypes.c_void_p), + ("type_k", ctypes.c_int), + ("type_v", ctypes.c_int), + ("abort_callback", ggml_abort_callback), + ("abort_callback_data", ctypes.c_void_p), + ("embeddings", ctypes.c_bool), + ("offload_kqv", ctypes.c_bool), + ("flash_attn", ctypes.c_bool), + ("no_perf", ctypes.c_bool), + ("op_offload", ctypes.c_bool), + ("swa_full", ctypes.c_bool), + ("kv_unified", ctypes.c_bool), + ("sampler", ctypes.c_void_p), + ("n_sampler", ctypes.c_int), + ("flash_attn_type", ctypes.c_int), # -1 LLAMA_FLASH_ATTN_TYPE_AUTO), + + ] + + +class llama_model_quantize_params(ctypes.Structure): + + _fields_ = [ + ("nthread", ctypes.c_int32), + ("ftype", ctypes.c_int), + ("output_tensor_type", ctypes.c_int), + ("token_embedding_type", ctypes.c_int), + ("allow_requantize", ctypes.c_bool), + ("quantize_output_tensor", ctypes.c_bool), + ("only_copy", ctypes.c_bool), + ("pure", ctypes.c_bool), + ("keep_split", ctypes.c_bool), + ("imatrix", ctypes.c_void_p), + ("kv_overrides", ctypes.c_void_p), + ("tensor_types", ctypes.c_void_p), + ("prune_layers", ctypes.c_void_p) + ] + + +class llama_logit_bias(ctypes.Structure): + + _fields_ = [ + ("token", llama_token), + ("bias", ctypes.c_float), + ] + +llama_logit_bias_p = ctypes.POINTER(llama_logit_bias) + + +class llama_grammar_element(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int), + ("value", ctypes.c_uint32), + ] + + +class llama_timings(ctypes.Structure): + _fields_ = [ + ("t_start_ms", ctypes.c_double), + ("t_end_ms", ctypes.c_double), + ("t_load_ms", ctypes.c_double), + ("t_sample_ms", ctypes.c_double), + ("t_p_eval_ms", ctypes.c_double), + ("t_eval_ms", ctypes.c_double), + ("n_sample", ctypes.c_int32), + ("n_p_eval", ctypes.c_int32), + ("n_eval", ctypes.c_int32), + ] + +class llama_chat_message(ctypes.Structure): + _fields_ = [ + ("role", ctypes.c_char_p), + ("content", ctypes.c_char_p), + ] + +class llama_kv_cache_view_cell(ctypes.Structure): + _fields_ = [("pos", llama_pos)] + +class llama_kv_cache_view(ctypes.Structure): + + _fields_ = [ + ("n_cells", ctypes.c_int32), + ("n_max_seq", ctypes.c_int32), + ("token_count", ctypes.c_int32), + ("used_cells", ctypes.c_int32), + ("max_contiguous", ctypes.c_int32), + ("max_contiguous_idx", ctypes.c_int32), + ("cells", ctypes.POINTER(llama_kv_cache_view_cell)), + ("cells_sequences", ctypes.POINTER(llama_seq_id)), + ] + +llama_kv_cache_view_p = ctypes.POINTER(llama_kv_cache_view) + +class llama_beam_view(ctypes.Structure): + _fields_ = [ + ("tokens", llama_token_p), + ("n_tokens", ctypes.c_size_t), + ("p", ctypes.c_float), + ("eob", ctypes.c_bool), + ] + + +class llama_beams_state(ctypes.Structure): + _fields_ = [ + ("beam_views", ctypes.POINTER(llama_beam_view)), + ("n_beams", ctypes.c_size_t), + ("common_prefix_length", ctypes.c_size_t), + ("last_call", ctypes.c_bool), + ] + + +llama_sampler_context_t = ctypes.c_void_p + + +class llama_sampler_i(ctypes.Structure): + ... + + +class llama_sampler_chain_params(ctypes.Structure): + + _fields_ = [ + ("no_perf", ctypes.c_bool), + ] + + +class llama_sampler(ctypes.Structure): + _fields_ = [ + ("iface", ctypes.POINTER(llama_sampler_i)), + ("ctx", llama_sampler_context_t), + ] + + +llama_sampler_p = ctypes.POINTER(llama_sampler) +llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler) + +llama_sampler_chain_get = ctypes.CFUNCTYPE(llama_sampler_p, llama_sampler_p_ctypes, ctypes.c_int32, llama_sampler_p_ctypes) +llama_sampler_chain_n = ctypes.CFUNCTYPE(ctypes.c_int, llama_sampler_p_ctypes) +llama_sampler_chain_remove = ctypes.CFUNCTYPE(llama_sampler_p , llama_sampler_p_ctypes, ctypes.c_int32, llama_sampler_p_ctypes) +llama_sampler_p_ctypes = ctypes.POINTER(llama_sampler) + +llama_sampler_i_name = ctypes.CFUNCTYPE(ctypes.c_char_p, llama_sampler_p_ctypes) +llama_sampler_i_accept = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token) +llama_sampler_i_apply = ctypes.CFUNCTYPE( + None, llama_sampler_p_ctypes, llama_token_data_array_p +) +llama_sampler_i_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_i_clone = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_sampler_p_ctypes) +llama_sampler_i_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) + +llama_sampler_i._fields_ = [ + ("name", llama_sampler_i_name), + ("accept", llama_sampler_i_accept), + ("apply", llama_sampler_i_apply), + ("reset", llama_sampler_i_reset), + ("clone", llama_sampler_i_clone), + ("free", llama_sampler_i_free), +] + +llama_sampler_name = ctypes.CFUNCTYPE(ctypes.c_char_p, llama_sampler_p_ctypes) +llama_sampler_accept = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token) +llama_sampler_apply = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes, llama_token_data_array_p) +llama_sampler_reset = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_clone = ctypes.CFUNCTYPE(llama_sampler_p, llama_sampler_p_ctypes) +llama_sampler_free = ctypes.CFUNCTYPE(None, llama_sampler_p_ctypes) +llama_sampler_init_dist = ctypes.CFUNCTYPE(llama_sampler_p, ctypes.c_uint32, llama_sampler_p_ctypes) + +llama_sampler_init_softmax = ctypes.CFUNCTYPE(llama_sampler_p, llama_sampler_p_ctypes) + +llama_sampler_init_top_k = ctypes.CFUNCTYPE(llama_sampler_p, ctypes.c_int32, llama_sampler_p_ctypes) + +llama_sampler_init_top_p = ctypes.CFUNCTYPE(llama_sampler_p, ctypes.c_float, ctypes.c_size_t) +llama_sampler_init_min_p = ctypes.CFUNCTYPE(llama_sampler_p, ctypes.c_float, ctypes.c_size_t) +llama_sampler_init_typical = ctypes.CFUNCTYPE(llama_sampler_p, ctypes.c_float, ctypes.c_size_t) + +llama_sampler_init_temp = ctypes.CFUNCTYPE(llama_sampler_p, ctypes.c_float, llama_sampler_p_ctypes) + +llama_sampler_init_temp_ext = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, ctypes.c_float, ctypes.c_float, ctypes.c_float) + +llama_sampler_init_xtc = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, ctypes.c_float, ctypes.c_float, ctypes.c_size_t, ctypes.c_uint32) + +llama_sampler_init_mirostat = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, ctypes.c_int32, ctypes.c_uint32, ctypes.c_float, + ctypes.c_float, ctypes.c_int32) + +llama_sampler_init_mirostat_v2 = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, ctypes.c_uint32, ctypes.c_float, ctypes.c_float) + +llama_sampler_init_grammar = ctypes.CFUNCTYPE(llama_sampler_p_ctypes, llama_model_p_ctypes, ctypes.c_char_p,ctypes.c_char_p) + + +def add_ctypes_declarations (_lib): + + """ Exposed methods on llama cpp binary as of January 2026 - roughly aligning to releases up to ~7900 """ + + llama_memory_clear = _lib.llama_memory_clear + llama_memory_clear.argtypes = [llama_memory_t_ctypes, ctypes.c_bool] + llama_memory_clear.restype = None + + llama_state_get_size = _lib.llama_state_get_size + llama_state_get_size.argtypes = [llama_context_p_ctypes] + llama_state_get_size.restype = ctypes.c_size_t + + llama_sampler_sample = _lib.llama_sampler_sample + llama_sampler_sample.argtypes = [llama_sampler_p_ctypes, llama_context_p_ctypes, ctypes.c_int32] + llama_sampler_sample.restype = llama_token + + llama_sampler_chain_init = _lib.llama_sampler_chain_init + llama_sampler_chain_init.argtypes = [llama_sampler_chain_params] + llama_sampler_chain_init.restype = llama_sampler_p_ctypes + + llama_sampler_chain_add = _lib.llama_sampler_chain_add + + # below is key fix for Mac - correcting the ctypes declaration for the arg types + # -- previously, alt/incorrect: [llama_sampler_p_ctypes] - only one arg + # -- incorrect declaration was OK on Windows and Linux + # -- correct declaration is two args both with same llama_sampler_p_ctypes + + llama_sampler_chain_add.argtypes = [llama_sampler_p_ctypes, llama_sampler_p_ctypes] + llama_sampler_chain_add.restype = None + + llama_sampler_init_greedy = _lib.llama_sampler_init_greedy + llama_sampler_init_greedy.argtypes = [] + llama_sampler_init_greedy.restype = llama_sampler_p + + # major interfaces + + llama_backend_init = _lib.llama_backend_init + llama_backend_init.argtypes = [] + llama_backend_init.restype = None + + llama_model_default_params = _lib.llama_model_default_params + llama_model_default_params.argtypes = [] + llama_model_default_params.restype = llama_model_params + + llama_context_default_params = _lib.llama_context_default_params + llama_context_default_params.argtypes = [] + llama_context_default_params.restype = llama_context_params + + llama_backend_free = _lib.llama_backend_free + llama_backend_free.argtypes = [] + llama_backend_free.restype = None + + llama_load_model_from_file = _lib.llama_load_model_from_file + llama_load_model_from_file.argtypes = [ctypes.c_char_p, llama_model_params] + llama_load_model_from_file.restype = llama_model_p_ctypes + + _lib.llama_max_devices.argtypes = [] + _lib.llama_max_devices.restype = ctypes.c_size_t + + llama_free_model = _lib.llama_free_model + llama_free_model.argtypes = [llama_model_p_ctypes] + llama_free_model.restype = None + + llama_init_from_model = _lib.llama_init_from_model + llama_init_from_model.argtypes = [llama_model_p_ctypes, llama_context_params] + llama_init_from_model.restype = llama_context_p_ctypes + + # deprecated in favor of llama_init_from_model + llama_new_context_with_model = _lib.llama_new_context_with_model + llama_new_context_with_model.argtypes = [llama_model_p_ctypes, llama_context_params] + llama_new_context_with_model.restype = llama_context_p_ctypes + + llama_free = _lib.llama_free + llama_free.argtypes = [llama_context_p_ctypes] + llama_free.restype = None + + llama_time_us = _lib.llama_time_us + llama_time_us.argtypes = [] + llama_time_us.restype = ctypes.c_int64 + + llama_max_devices = _lib.llama_max_devices + llama_max_devices.argtypes = [] + llama_max_devices.restype = ctypes.c_size_t + + llama_supports_mmap = _lib.llama_supports_mmap + llama_supports_mmap.argtypes = [] + llama_supports_mmap.restype = ctypes.c_bool + + llama_supports_mlock = _lib.llama_supports_mlock + llama_supports_mlock.argtypes = [] + llama_supports_mlock.restype = ctypes.c_bool + + llama_supports_gpu_offload = _lib.llama_supports_gpu_offload + llama_supports_gpu_offload.argtypes = [] + llama_supports_gpu_offload.restype = ctypes.c_bool + + llama_get_model = _lib.llama_get_model + llama_get_model.argtypes = [llama_context_p_ctypes] + llama_get_model.restype = llama_model_p_ctypes + + llama_n_ctx = _lib.llama_n_ctx + llama_n_ctx.argtypes = [llama_context_p_ctypes] + llama_n_ctx.restype = ctypes.c_uint32 + + llama_n_batch = _lib.llama_n_batch + llama_n_batch.argtypes = [llama_context_p_ctypes] + llama_n_batch.restype = ctypes.c_uint32 + + llama_vocab_type = _lib.llama_vocab_type + llama_vocab_type.argtypes = [llama_model_p_ctypes] + llama_vocab_type.restype = ctypes.c_int + + llama_n_vocab = _lib.llama_n_vocab + llama_n_vocab.argtypes = [ctypes.c_void_p] # [llama_model_p_ctypes] + llama_n_vocab.restype = ctypes.c_int32 + + llama_model_get_vocab = _lib.llama_model_get_vocab + llama_model_get_vocab.argtypes = [llama_model_p_ctypes] + llama_model_get_vocab.restype = llama_vocab_p_ctypes + + llama_n_ctx_train = _lib.llama_n_ctx_train + llama_n_ctx_train.argtypes = [llama_model_p_ctypes] + llama_n_ctx_train.restype = ctypes.c_int32 + + llama_n_embd = _lib.llama_n_embd + llama_n_embd.argtypes = [llama_model_p_ctypes] + llama_n_embd.restype = ctypes.c_int32 + + llama_model_meta_val_str = _lib.llama_model_meta_val_str + llama_model_meta_val_str.argtypes = [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_size_t, ] + llama_model_meta_val_str.restype = ctypes.c_int32 + + llama_model_meta_count = _lib.llama_model_meta_count + llama_model_meta_count.argtypes = [llama_model_p_ctypes] + llama_model_meta_count.restype = ctypes.c_int32 + + llama_model_meta_key_by_index = _lib.llama_model_meta_key_by_index + llama_model_meta_key_by_index.argtypes = [llama_model_p_ctypes, ctypes.c_int32, ctypes.c_char_p, ctypes.c_size_t, ] + llama_model_meta_key_by_index.restype = ctypes.c_int32 + + llama_model_meta_val_str_by_index = _lib.llama_model_meta_val_str_by_index + llama_model_meta_val_str_by_index.argtypes = [llama_model_p_ctypes, ctypes.c_int32, ctypes.c_char_p, + ctypes.c_size_t, ] + llama_model_meta_val_str_by_index.restype = ctypes.c_int32 + + llama_model_desc = _lib.llama_model_desc + llama_model_desc.argtypes = [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_size_t] + llama_model_desc.restype = ctypes.c_int32 + + llama_model_size = _lib.llama_model_size + llama_model_size.argtypes = [llama_model_p_ctypes] + llama_model_size.restype = ctypes.c_uint64 + + llama_model_n_params = _lib.llama_model_n_params + llama_model_n_params.argtypes = [llama_model_p_ctypes] + llama_model_n_params.restype = ctypes.c_uint64 + + llama_memory_seq_rm = _lib.llama_memory_seq_rm + llama_memory_seq_rm.argtypes = [llama_memory_t_ctypes, llama_seq_id, llama_pos, llama_pos,] + llama_memory_seq_rm.restype = ctypes.c_bool + + llama_get_memory = _lib.llama_get_memory + llama_get_memory.argtypes = [llama_context_p_ctypes] + llama_get_memory.restype = ctypes.c_void_p + + llama_batch_init = _lib.llama_batch_init + llama_batch_init.argtypes = [ctypes.c_int32, ctypes.c_int32, ctypes.c_int32] + llama_batch_init.restype = llama_batch + + llama_batch_free = _lib.llama_batch_free + llama_batch_free.argtypes = [llama_batch] + llama_batch_free.restype = None + + llama_decode = _lib.llama_decode + llama_decode.argtypes = [llama_context_p_ctypes, llama_batch] + llama_decode.restype = ctypes.c_int32 + + llama_set_n_threads = _lib.llama_set_n_threads + llama_set_n_threads.argtypes = [llama_context_p_ctypes, ctypes.c_uint32, ctypes.c_uint32, ] + llama_set_n_threads.restype = None + + llama_get_logits = _lib.llama_get_logits + llama_get_logits.argtypes = [llama_context_p_ctypes] + llama_get_logits.restype = ctypes.POINTER(ctypes.c_float) + + llama_get_logits_ith = _lib.llama_get_logits_ith + llama_get_logits_ith.argtypes = [llama_context_p_ctypes, ctypes.c_int32] + llama_get_logits_ith.restype = ctypes.POINTER(ctypes.c_float) + + llama_get_embeddings = _lib.llama_get_embeddings + llama_get_embeddings.argtypes = [llama_context_p_ctypes] + llama_get_embeddings.restype = ctypes.POINTER(ctypes.c_float) + + llama_get_embeddings_ith = _lib.llama_get_embeddings_ith + llama_get_embeddings_ith.argtypes = [llama_context_p_ctypes, ctypes.c_int32] + llama_get_embeddings_ith.restype = ctypes.POINTER(ctypes.c_float) + + llama_token_get_text = _lib.llama_token_get_text + llama_token_get_text.argtypes = [llama_model_p_ctypes, llama_token] + llama_token_get_text.restype = ctypes.c_char_p + + llama_token_get_score = _lib.llama_token_get_score + llama_token_get_score.argtypes = [llama_model_p_ctypes, llama_token] + llama_token_get_score.restype = ctypes.c_float + + llama_token_bos = _lib.llama_token_bos + llama_token_bos.argtypes = [llama_model_p_ctypes] + llama_token_bos.restype = llama_token + + llama_token_eos = _lib.llama_token_eos + llama_token_eos.argtypes = [llama_model_p_ctypes] + llama_token_eos.restype = llama_token + + llama_token_nl = _lib.llama_token_nl + llama_token_nl.argtypes = [llama_model_p_ctypes] + llama_token_nl.restype = llama_token + + llama_add_bos_token = _lib.llama_add_bos_token + llama_add_bos_token.argtypes = [llama_model_p_ctypes] + llama_add_bos_token.restype = ctypes.c_int32 + + llama_add_eos_token = _lib.llama_add_eos_token + llama_add_eos_token.argtypes = [llama_model_p_ctypes] + llama_add_eos_token.restype = ctypes.c_int32 + + llama_token_eot = _lib.llama_token_eot + llama_token_eot.argtypes = [llama_model_p_ctypes] + llama_token_eot.restype = llama_token + + llama_tokenize = _lib.llama_tokenize + llama_tokenize.argtypes = [llama_model_p_ctypes, ctypes.c_char_p, ctypes.c_int32, llama_token_p, ctypes.c_int32, + ctypes.c_bool, ctypes.c_bool, ] + llama_tokenize.restype = ctypes.c_int32 + + llama_token_to_piece = _lib.llama_token_to_piece + llama_token_to_piece.argtypes = [llama_model_p_ctypes, llama_token, ctypes.c_char_p, ctypes.c_int32, ] + llama_token_to_piece.restype = ctypes.c_int32 + + llama_grammar_element_p = ctypes.POINTER(llama_grammar_element) + + llama_print_system_info = _lib.llama_print_system_info + llama_print_system_info.argtypes = [] + llama_print_system_info.restype = ctypes.c_char_p + + llama_log_set = _lib.llama_log_set + llama_log_set.argtypes = [ctypes.c_void_p, ctypes.c_void_p] + llama_log_set.restype = None + + return _lib + + +class _LlamaModel: + + """ _LLamaModel is a Python object wrapper around the C pointer to the llama_cpp model object + that is created upon loading. It does not do much, except provide a 'home' to self.model, which + points to the loaded gguf model and is used in all methods. """ + + _llama_free_model = None + + def __init__(self, _lib, path_model, params): + + self.path_model = path_model + self.params = params + + self._llama_free_model = _lib.llama_free_model + + self.model = None + self.sampler = None + + if not os.path.exists(path_model): + pass + + # main function call to _lib + self.model = _lib.llama_load_model_from_file(self.path_model.encode("utf-8"), self.params) + + if self.model is None: + pass + + def __del__(self): + if self.model is not None and self._llama_free_model is not None: + self._llama_free_model(self.model) + self.model = None + + +class _LlamaContext: + + """ _LlamaContext is a Python object wrapper around the context object pointer instantiated by llama.cpp. + The context consists of a combination of a model and set of sampling parameters. This object does not + do much, except provide the home to self.ctx which is the context object used for all methods. """ + + _llama_free = None + + def __init__(self, _lib, model, params): + + self.model = model + self.params = params + + self.verbose = True + self.sampler = None + + self._llama_free = _lib.llama_free + + assert self.model.model is not None + + self.ctx = _lib.llama_init_from_model(self.model.model, self.params) + + self.memory = _lib.llama_get_memory(self.ctx) + + if self.ctx is None: + pass + + def __del__(self): + if self.ctx is not None and self._llama_free is not None: + self._llama_free(self.ctx) + self.ctx = None + + +class _LlamaBatch: + + """ _LLamaBatch object is a Python object wrapper around llama_cpp batch object pointer, which is + used in the generation process. """ + + # largely follows implementation from llama-cpp-python + + _llama_batch_free = None + + def __init__(self, _lib, n_tokens, embd, n_seq_max): + + self._n_tokens = n_tokens + self.embd = embd + self.n_seq_max = n_seq_max + + self._llama_batch_free = _lib.llama_batch_free + + self.batch = None + self.batch = _lib.llama_batch_init(self._n_tokens, self.embd, self.n_seq_max) + + def __del__(self): + if self.batch is not None and self._llama_batch_free is not None: + self._llama_batch_free(self.batch) + self.batch = None + + def n_tokens(self): + assert self.batch is not None + return self.batch.n_tokens + + def reset(self): + assert self.batch is not None + self.batch.n_tokens = 0 + + def set_batch(self, batch, n_past, logits_all): + assert self.batch is not None + n_tokens = len(batch) + self.batch.n_tokens = n_tokens + for i in range(n_tokens): + self.batch.token[i] = batch[i] + self.batch.pos[i] = n_past + i + self.batch.seq_id[i][0] = 0 + self.batch.n_seq_id[i] = 1 + self.batch.logits[i] = logits_all + + self.batch.logits[n_tokens - 1] = True + + def add_sequence(self, batch, seq_id, logits_all): + assert self.batch is not None + n_tokens = len(batch) + n_tokens0 = self.batch.n_tokens + self.batch.n_tokens += n_tokens + for i in range(n_tokens): + j = n_tokens0 + i + self.batch.token[j] = batch[i] + self.batch.pos[j] = i + self.batch.seq_id[j][0] = seq_id + self.batch.n_seq_id[j] = 1 + self.batch.logits[j] = logits_all + self.batch.logits[n_tokens - 1] = True + + +class _LlamaTokenDataArray: + + """_LlamaTokenDataArray is a Python object wrapper around llama_cpp token data array object, which is used as + input into the generation inference process. """ + + # follows the implementation from llama-cpp-python + + def __init__(self, *, n_vocab): + self.n_vocab = n_vocab + self.candidates_data = np.array( + [], + dtype=np.dtype( + [("id", np.intc), ("logit", np.single), ("p", np.single)], align=True + ), + ) + self.candidates_data.resize(3, self.n_vocab, refcheck=False) + self.candidates = llama_token_data_array( + data=self.candidates_data.ctypes.data_as(llama_token_data_p), + size=self.n_vocab, + sorted=False, + ) + self.default_candidates_data_id = np.arange(self.n_vocab, dtype=np.intc) + self.default_candidates_data_p = np.zeros(self.n_vocab, dtype=np.single) + + def copy_logits(self, logits): + self.candidates_data["id"][:] = self.default_candidates_data_id + self.candidates_data["logit"][:] = logits + self.candidates_data["p"][:] = self.default_candidates_data_p + self.candidates.data = self.candidates_data.ctypes.data_as(llama_token_data_p) + self.candidates.sorted = ctypes.c_bool(False) + self.candidates.size = ctypes.c_size_t(self.n_vocab) + + +@llama_log_callback +def llama_log_callback(level, text, user_data): + + """ Controls the display log output from llama.cpp engine - currently exposing two options 'ON' or 'OFF' """ + + # note: reserving level and user_data as options for the future + # --adapted from more sophisticated logging mechanism in llama-cpp-python + + if os.environ.get("llama_cpp_verbose") != "OFF": + print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) + else: + # no action taken if verbose is if OFF + do_nothing = 0 + +@whisper_log_callback +def whisper_log_callback(level, text, user_data): + + """ Controls the display log output from llama.cpp engine - currently exposing two options 'ON' or 'OFF' """ + + # note: reserving level and user_data as options for the future + # --adapted from more sophisticated logging mechanism in llama-cpp-python + + if os.environ.get("whisper_cpp_verbose") != "OFF": + print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) + else: + # no action taken if verbose is if OFF + do_nothing = 0 + + +mtmd_log_callback = ctypes.CFUNCTYPE(None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p) + + +@mtmd_log_callback +def mtmd_log_callback(level, text, user_data): + + """ Controls the display log output from mtmd engine - currently exposing two options 'ON' or 'OFF' """ + + # note: reserving level and user_data as options for the future + # --adapted from more sophisticated logging mechanism in llama-cpp-python + # --integrated with llama_cpp_verbose logging option for integrated debugging + + if os.environ.get("llama_cpp_verbose") != "OFF": + print(text.decode("utf-8"), end="", flush=True, file=sys.stderr) + else: + # no action taken if verbose is if OFF + do_nothing = 0 + + +class GGUFConfigs: + + """GGUFConfigs is main global configuration object for GGUF Generative Models. Most of these config items + do not need to be changed - and should be changed only if you know why you are changing them, as it could + impact the stability of the back-end llama.cpp library. + + The most common configs are exposed in _conf_libs + + """ + + # note: to "bring your own" llama.cpp custom compiled back-end, set the following: + # GGUFConfigs().set_config("custom_lib_path") = "/path/to/your/lib" + + _conf_libs = {"custom_lib_path": None, + + # --Mac: uses Mac Metal GPU by default + # --Linux / Windows - checks for cuda availability + "use_gpu": True, + + # note this will be used on Windows and Linux, but not Mac + "n_gpu_layers": 50, + "cuda_driver_min_level": 12.1, + "cuda_platforms": ["linux", "win32"], + "backend_initialized": False, + + # min cuda drivers for build of cuda libs + "cuda_linux_driver_min": [525, 60], + "cuda_windows_driver_min": [528,33], + + # adjusted from 256 (default for a long time - too low) + "max_output_tokens": 2048, + "temperature_default": 0.3, + + "llama_cpp_verbose": "OFF", + "force_gpu": False, + "use_macos_accelerate": True, + + # option to capture and provide the 'first token' of generation + # used for GGUF - and implemented for HFGenerative (Pytorch) and + # ONNXGenerative classes as well + "get_first_token_speed": False, + + # prebuilt shared libraries included in llmware + "windows_x86_lib": "gguf_win_x86", + "windows_cuda_lib": "gguf_win_cuda", + "windows_arm64_lib": "gguf_win_arm64", + + "linux_x86_lib": "gguf_linux_x86", + "linux_cuda_lib": "gguf_linux_cuda", + + # dgx = linux aarch64 cuda + "linux_aarch64_cuda_lib": "gguf_dgx", + + "mac_metal_lib": "gguf_mac", + + # prebuilt binaries packaged with llmware - evolving over time + + "windows": "llama.dll", + "windows_cuda": "llama.dll", + "windows_arm64": "llama.dll", + "mac_metal": "libllama.dylib", + "linux_x86": "libllama.so", + "linux_cuda": "libllama.so", + + # removed/deprecated support for older M-series Macs without Accelerate + # "mac_metal_no_acc": "libllama.dylib", + + "windows_mtmd": "mtmd.dll", + "mac_metal_mtmd": "libmtmd.dylib", + "linux_x86_mtmd": "libmtmd.so", + "linux_cuda_mtmd": "libmtmd.so", + "windows_arm64_mtmd": "mtmd.dll", + "windows_cuda_mtmd": "mtmd.dll", + + "n_threads": 6, # max(multiprocessing.cpu_count() // 2, 1), + "n_threads_batch": 6, # max(multiprocessing.cpu_count() // 2, 1), + + # whisper cpp configs + "whisper_cpp_lib_path": None, + "whisper_cpp_verbose": "OFF", + + # turn on for testing/debugging + "whisper_cpp_realtime_display": False, + "whisper_language": "en", + "whisper_sr": 16000, + "whisper_strategy": 0, + "whisper_threads": 2, + "whisper_beam_size": 5, + "whisper_greedy_best_of": 5, + "whisper_temperature_inc": 0.4, + "whisper_tiny_diarize": True, + "whisper_remove_segment_markers": False, + "whisper_output_format": "text", + "whisper_default_model": "whisper-cpp-base-english", + + # option to continue using older whisper cpp library on mac + "whisper_use_legacy_mac": True, + # prebuilt shared libraries included in llmware - evolving over time + "whisper_dgx": "libwhisper.so", + "whisper_linux_cuda": "libwhisper.so", + "whisper_linux_x86": "libwhisper.so", + "whisper_mac_metal": "libwhisper.dylib", + "whisper_mac_metal_legacy": "libwhisper_mac_metal_155.dylib", + "whisper_windows": "whisper.dll", + "whisper_windows_arm64": "whisper.dll", + } + + # note: with temperature used as primary attribute to adjust sampling, + # most of the params do not need to be adjusted + + _conf_sampling_params = {"top_k": 40, + "top_p": 0.95, + "min_p": 0.05, + "tfs_z": 1.0, + "typical_p": 1.0, + "penalty_last_n": 64, + "penalty_repeat": 1.1, + "penalty_freq": 0.0, + "penalty_present": 0.0, + "mirostat": 0, + "mirostat_tau": 5.0, + "mirostat_eta": 0.1, + "penalize_nl": True, + "logit_bias": {}, + "cfg_scale": 1.0, + "n_probs": 0, + "mirostat_mu": field(default_factory=ctypes.c_float) + } + + _conf_context_params = {"seed": 0xFFFFFFFF, + "n_ctx": 2048, + "n_batch": 2048, + "n_threads": 1, # check/confirm + "n_threads_batch": 1, # check/confirm + "rope_scaling_type": -1, + "rope_freq_base": 0.0, + "rope_freq_scale": 0.0, + "yarn_ext_factor": -1.0, + "yarn_attn_factor": 1.0, + "yarn_beta_fast": 32.0, + "yarn_beta_slow":1.0, + "yarn_orig_ctx": 0, + "mul_mat_q": True, + "logits_all": False, + "embedding": False, + "offload_kqv": True + } + + @classmethod + def get_config(cls, name): + if name in cls._conf_libs: + return cls._conf_libs[name] + + @classmethod + def set_config(cls, name, value): + cls._conf_libs[name] = value + + @classmethod + def get_sampling_params(cls): + return cls._conf_sampling_params + + +# *** WHISPER CPP CONFIGS START HERE *** + + +class whisper_token_data(ctypes.Structure): + _fields_ = [ + ("id", ctypes.c_int), + ("tid", ctypes.c_int), + ("p", ctypes.c_float), + ("plog", ctypes.c_float), + ("pt", ctypes.c_float), + ("ptsum", ctypes.c_float), + ("t0", ctypes.c_int64), + ("t1", ctypes.c_int64), + + # new param + ("t_dtw", ctypes.c_int64), + + ("vlen", ctypes.c_float), + ] + + +whisper_new_segment_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) + +whisper_progress_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p) + +whisper_encoder_begin_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p) + +abort_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p) + +whisper_logits_filter_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p, + ctypes.POINTER(whisper_token_data), ctypes.c_int, + ctypes.POINTER(ctypes.c_float), ctypes.c_void_p) + + +class greedy(ctypes.Structure): + _fields_ = [ + ("best_of", ctypes.c_int), + ] + + +class beam_search(ctypes.Structure): + _fields_ = [ + ("beam_size", ctypes.c_int), + ("patience", ctypes.c_float), + ] + + +class whisper_grammar_element(ctypes.Structure): + + _fields_ = [ + ("whisper_gretype", ctypes.c_int64), + ("type", ctypes.c_uint32) + ] + + + +class whisper_vad_default_params(ctypes.Structure): + + _fields_ = [ + ("threshold", ctypes.c_float), + ("min_speech_duration_ms", ctypes.c_float), + ("min_silence_duration_ms", ctypes.c_float), + ("max_speech_duration_ms", ctypes.c_float), + ("speech_pad_ms", ctypes.c_float), + ("samples_overlap", ctypes.c_float) + ] + + +class whisper_full_params(ctypes.Structure): + + _fields_ = [ + ("strategy", ctypes.c_int), + ("n_threads", ctypes.c_int), + ("n_max_text_ctx", ctypes.c_int), + ("offset_ms", ctypes.c_int), + ("duration_ms", ctypes.c_int), + ("translate", ctypes.c_bool), + ("no_context", ctypes.c_bool), + ("no_timestamps", ctypes.c_bool), + ("single_segment", ctypes.c_bool), + ("print_special", ctypes.c_bool), + ("print_progress", ctypes.c_bool), + ("print_realtime", ctypes.c_bool), + ("print_timestamps", ctypes.c_bool), + ("token_timestamps", ctypes.c_bool), + ("thold_pt", ctypes.c_float), + ("thold_ptsum", ctypes.c_float), + ("max_len", ctypes.c_int), + ("split_on_word", ctypes.c_bool), + ("max_tokens", ctypes.c_int), + ("debug_mode", ctypes.c_bool), + ("audio_ctx", ctypes.c_int), + ("tdrz_enable", ctypes.c_bool), + ("suppress_regex", ctypes.c_char_p), + ("initial_prompt", ctypes.c_char_p), + ("prompt_tokens", ctypes.POINTER(ctypes.c_int)), + ("prompt_n_tokens", ctypes.c_int), + ("language", ctypes.c_char_p), + ("detect_language", ctypes.c_bool), + ("suppress_blank", ctypes.c_bool), + ("suppress_nst", ctypes.c_bool), + ("temperature", ctypes.c_float), + ("max_initial_ts", ctypes.c_float), + ("length_penalty", ctypes.c_float), + ("temperature_inc", ctypes.c_float), + ("entropy_thold", ctypes.c_float), + ("logprob_thold", ctypes.c_float), + ("no_speech_thold", ctypes.c_float), + ("greedy", greedy), + ("beam_search", beam_search), + ("new_segment_callback", whisper_new_segment_callback), + ("new_segment_callback_user_data", ctypes.c_void_p), + ("progress_callback", whisper_progress_callback), + ("progress_callback_user_data", ctypes.c_void_p), + ("encoder_begin_callback", whisper_encoder_begin_callback), + ("encoder_begin_callback_user_data", ctypes.c_void_p), + ("abort_callback", abort_callback), + ("abort_callback_user_data", ctypes.c_void_p), + ("logits_filter_callback", whisper_logits_filter_callback), + ("logits_filter_callback_user_data", ctypes.c_void_p), + ("grammar_rules", whisper_grammar_element), + ("n_grammar_rules", ctypes.c_size_t), + ("i_start_rule", ctypes.c_size_t), + ("grammar_penalty", ctypes.c_float), + ("vad", ctypes.c_bool), + ("vad_model_path", ctypes.c_char_p), + ("vad_params", whisper_vad_default_params) + ] + + +class whisper_full_params_legacy(ctypes.Structure): + + _fields_ = [ + ("strategy", ctypes.c_int), + ("n_threads", ctypes.c_int), + ("n_max_text_ctx", ctypes.c_int), + ("offset_ms", ctypes.c_int), + ("duration_ms", ctypes.c_int), + ("translate", ctypes.c_bool), + ("no_context", ctypes.c_bool), + ("no_timestamps", ctypes.c_bool), + ("single_segment", ctypes.c_bool), + ("print_special", ctypes.c_bool), + ("print_progress", ctypes.c_bool), + ("print_realtime", ctypes.c_bool), + ("print_timestamps", ctypes.c_bool), + ("token_timestamps", ctypes.c_bool), + ("thold_pt", ctypes.c_float), + ("thold_ptsum", ctypes.c_float), + ("max_len", ctypes.c_int), + ("split_on_word", ctypes.c_bool), + ("max_tokens", ctypes.c_int), + + # speed_up removed in later versions + ("speed_up", ctypes.c_bool), + + ("debug_mode", ctypes.c_bool), + ("audio_ctx", ctypes.c_int), + ("tdrz_enable", ctypes.c_bool), + ("suppress_regex", ctypes.c_char_p), + ("initial_prompt", ctypes.c_char_p), + ("prompt_tokens", ctypes.POINTER(ctypes.c_int)), + ("prompt_n_tokens", ctypes.c_int), + ("language", ctypes.c_char_p), + ("detect_language", ctypes.c_bool), + ("suppress_blank", ctypes.c_bool), + + # update new versions: suppress_non_speech_tokens -> suppress_nst + ("suppress_non_speech_tokens", ctypes.c_bool), + + ("temperature", ctypes.c_float), + ("max_initial_ts", ctypes.c_float), + ("length_penalty", ctypes.c_float), + ("temperature_inc", ctypes.c_float), + ("entropy_thold", ctypes.c_float), + ("logprob_thold", ctypes.c_float), + ("no_speech_thold", ctypes.c_float), + ("greedy", greedy), + ("beam_search", beam_search), + ("new_segment_callback", whisper_new_segment_callback), + ("new_segment_callback_user_data", ctypes.c_void_p), + ("progress_callback", whisper_progress_callback), + ("progress_callback_user_data", ctypes.c_void_p), + ("encoder_begin_callback", whisper_encoder_begin_callback), + ("encoder_begin_callback_user_data", ctypes.c_void_p), + ("abort_callback", abort_callback), + ("abort_callback_user_data", ctypes.c_void_p), + ("logits_filter_callback", whisper_logits_filter_callback), + ("logits_filter_callback_user_data", ctypes.c_void_p), + ("grammar_rules", whisper_grammar_element), + ("n_grammar_rules", ctypes.c_size_t), + ("i_start_rule", ctypes.c_size_t), + ("grammar_penalty", ctypes.c_float), + + # new parameters added later: + # ("vad", ctypes.c_bool), + # ("vad_model_path", ctypes.c_char_p), + # ("vad_params", whisper_vad_default_params) + ] + + +""" MTMD & CLIP GGUF Interface Configurations """ + +mtmd_context_p = NewType("mtmd_context_p", int) +mtmd_context_p_ctypes = ctypes.c_void_p + +mtmd_bitmap_p = NewType("mtmd_bitmap_p", int) +mtmd_bitmap_p_ctypes = ctypes.c_void_p + +mtmd_image_tokens_p = NewType("mtmd_image_tokens_p", int) +mtmd_image_tokens_p_ctypes = ctypes.c_void_p + +mtmd_input_chunk_p = NewType("mtmd_input_chunk_p", int) +mtmd_input_chunk_p_ctypes = ctypes.c_void_p + +mtmd_input_chunks_p = NewType("mtmd_input_chunks_p", int) +mtmd_input_chunks_p_ctypes = ctypes.c_void_p + +MTMD_INPUT_CHUNK_TYPE_TEXT = 0 +MTMD_INPUT_CHUNK_TYPE_IMAGE = 1 +MTMD_INPUT_CHUNK_TYPE_AUDIO = 2 + + +class mtmd_context_params(ctypes.Structure): + + """ This interface is linked to mtmd with releases b7062+, e.g., starting ~Nov 2025 """ + + # if errors, look at this interface in llama.cpp/tools/mtmd/mtmd.h + # -- this api has been evolving + # -- see also class below as drop-in replacement if using a mtmd lib from before Nov 2025 + + _fields_ = [ + ("use_gpu", ctypes.c_bool), + ("print_timings", ctypes.c_bool), + ("n_threads", ctypes.c_int), + ("image_marker", ctypes.c_char_p), + ("media_marker", ctypes.c_char_p), + + # verbosity removed in b7062 + # ("verbosity", ctypes.c_int), # ggml_log_level + + # new starting b6935 + ("llama_flash_attn_type", ctypes.c_int), + ("warmup", ctypes.c_bool), + ("image_min_tokens", ctypes.c_int), + ("image_max_tokens", ctypes.c_int), + ("cb_eval_user_data", ctypes.c_void_p), + ("cb_eval", ggml_backend_sched_eval_callback) + ] + + +class mtmd_context_params_alt_pre7062 (ctypes.Structure): + + """ This is a deprecated interface that maps to mtmd releases in second half of 2025, up + to the b7062 release in November 2025 """ + + _fields_ = [ + ("use_gpu", ctypes.c_bool), + ("print_timings", ctypes.c_bool), + ("n_threads", ctypes.c_int), + ("verbosity", ctypes.c_int), # ggml_log_level + ("image_marker", ctypes.c_char_p), + ("media_marker", ctypes.c_char_p) + ] + + +class mtmd_input_text(ctypes.Structure): + _fields_ = [ + ("text", ctypes.c_char_p), + ("add_special", ctypes.c_bool), + ("parse_special", ctypes.c_bool), + ] + + +def add_libmtmd_ctypes_declarations(_libmtmd): + + """ Main mtmd library interfaces """ + + mtmd_default_marker = _libmtmd.mtmd_default_marker + mtmd_default_marker.argtypes = [] + mtmd_default_marker.restype = ctypes.c_char_p + + mtmd_context_params_default = _libmtmd.mtmd_context_params_default + mtmd_context_params_default.argtypes = [] + mtmd_context_params_default.restype = mtmd_context_params + + mtmd_init_from_file = _libmtmd.mtmd_init_from_file + mtmd_init_from_file.argtypes = [ctypes.c_char_p, llama_model_p_ctypes, mtmd_context_params] + mtmd_init_from_file.restype = mtmd_context_p_ctypes + + mtmd_free = _libmtmd.mtmd_free + mtmd_free.argtypes = [mtmd_context_p_ctypes] + mtmd_free.restype = None + + mtmd_support_vision = _libmtmd.mtmd_support_vision + mtmd_support_vision.argtypes = [mtmd_context_p_ctypes] + mtmd_support_vision.restype = ctypes.c_bool + + mtmd_bitmap_init = _libmtmd.mtmd_bitmap_init + mtmd_bitmap_init.argtypes = [ctypes.c_uint32, ctypes.c_uint32, ctypes.POINTER(ctypes.c_uint8)] + mtmd_bitmap_init.restype = mtmd_bitmap_p_ctypes + + mtmd_bitmap_free = _libmtmd.mtmd_bitmap_free + mtmd_bitmap_free.argtypes = [mtmd_bitmap_p_ctypes] + mtmd_bitmap_free.restype = None + + mtmd_input_chunks_init = _libmtmd.mtmd_input_chunks_init + mtmd_input_chunks_init.argtypes = [] + mtmd_input_chunks_init.restype = mtmd_input_chunks_p_ctypes + + mtmd_input_chunks_free = _libmtmd.mtmd_input_chunks_free + mtmd_input_chunks_free.argtypes = [mtmd_input_chunks_p_ctypes] + mtmd_input_chunks_free.restype = None + + mtmd_input_chunks_size = _libmtmd.mtmd_input_chunks_size + mtmd_input_chunks_size.argtypes = [mtmd_input_chunks_p_ctypes] + mtmd_input_chunks_size.restype = ctypes.c_size_t + + mtmd_input_chunks_get = _libmtmd.mtmd_input_chunks_get + mtmd_input_chunks_get.argtypes = [mtmd_input_chunks_p_ctypes, ctypes.c_size_t] + mtmd_input_chunks_get.restype = mtmd_input_chunk_p_ctypes + + mtmd_tokenize = _libmtmd.mtmd_tokenize + mtmd_tokenize.argtypes = [mtmd_context_p_ctypes, mtmd_input_chunks_p_ctypes, + ctypes.POINTER(mtmd_input_text), ctypes.POINTER(mtmd_bitmap_p_ctypes), + ctypes.c_size_t] + mtmd_tokenize.restype = ctypes.c_int + + mtmd_input_chunk_get_n_tokens = _libmtmd.mtmd_input_chunk_get_n_tokens + mtmd_input_chunk_get_n_tokens.argtypes = [mtmd_input_chunk_p_ctypes] + mtmd_input_chunk_get_n_tokens.restype = ctypes.c_size_t + + mtmd_input_chunk_get_type = _libmtmd.mtmd_input_chunk_get_type + mtmd_input_chunk_get_type.argtypes = [mtmd_input_chunk_p_ctypes] + mtmd_input_chunk_get_type.restype = ctypes.c_int + + mtmd_input_chunk_get_tokens_text = _libmtmd.mtmd_input_chunk_get_tokens_text + mtmd_input_chunk_get_tokens_text.argtypes = [mtmd_input_chunk_p_ctypes, ctypes.POINTER(ctypes.c_size_t)] + mtmd_input_chunk_get_tokens_text.restype = ctypes.POINTER(llama_token) + + # mtmd_helper_bitmap_init_from_buf + mtmd_helper_bitmap_init_from_buf = _libmtmd.mtmd_helper_bitmap_init_from_buf + mtmd_helper_bitmap_init_from_buf.argtypes = [mtmd_context_p_ctypes, ctypes.POINTER(ctypes.c_uint8), ctypes.c_size_t] + mtmd_helper_bitmap_init_from_buf.restype = mtmd_bitmap_p_ctypes + + # mtmd_helper_bitmap_init_from_file(mtmd_context * ctx, const char * fname) + mtmd_helper_bitmap_init_from_file = _libmtmd.mtmd_helper_bitmap_init_from_file + mtmd_helper_bitmap_init_from_file.argtypes = [mtmd_context_p_ctypes, ctypes.c_char_p] + mtmd_helper_bitmap_init_from_file.restype = mtmd_bitmap_p_ctypes + + mtmd_helper_get_n_tokens = _libmtmd.mtmd_helper_get_n_tokens + mtmd_helper_get_n_tokens.argtypes = [mtmd_input_chunks_p_ctypes] + mtmd_helper_get_n_tokens.restype = ctypes.c_size_t + + mtmd_helper_eval_chunk_single = _libmtmd.mtmd_helper_eval_chunk_single + mtmd_helper_eval_chunk_single.argtypes = [mtmd_context_p_ctypes, + llama_context_p_ctypes, + mtmd_input_chunk_p_ctypes, + llama_pos, llama_seq_id, + ctypes.c_int, ctypes.c_bool, ctypes.POINTER(llama_pos)] + mtmd_helper_eval_chunk_single.restype = ctypes.c_int + + # expose mtmd_helper_log_set - but catch if not found + + try: + mtmd_helper_log_set = _libmtmd.mtmd_helper_log_set + mtmd_helper_log_set.argtypes = [ctypes.c_void_p, ctypes.c_void_p] + mtmd_helper_log_set.restype = None + except: + pass + + return _libmtmd + diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libbson-1.0.0.0.0.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libbson-1.0.0.0.0.dylib new file mode 100755 index 0000000..3ee1b6e Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libbson-1.0.0.0.0.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/liblzma.5.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/liblzma.5.dylib new file mode 100644 index 0000000..2199dfc Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/liblzma.5.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libmongoc-1.0.0.0.0.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libmongoc-1.0.0.0.0.dylib new file mode 100755 index 0000000..0d8a3ed Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libmongoc-1.0.0.0.0.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libpng16.16.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libpng16.16.dylib new file mode 100755 index 0000000..f6c470f Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libpng16.16.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libpq.5.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libpq.5.dylib new file mode 100755 index 0000000..d45a119 Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libpq.5.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libsnappy.1.1.10.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libsnappy.1.1.10.dylib new file mode 100644 index 0000000..489f49b Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libsnappy.1.1.10.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libsqlite3.0.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libsqlite3.0.dylib new file mode 100755 index 0000000..053dc49 Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libsqlite3.0.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libxml2.2.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libxml2.2.dylib new file mode 100755 index 0000000..179e8ed Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libxml2.2.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libzip.5.5.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libzip.5.5.dylib new file mode 100755 index 0000000..515fc62 Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libzip.5.5.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/.dylibs/libzstd.1.5.5.dylib b/llmware/lib/darwin/arm64/llmware/.dylibs/libzstd.1.5.5.dylib new file mode 100644 index 0000000..40241d9 Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/.dylibs/libzstd.1.5.5.dylib differ diff --git a/llmware/lib/darwin/arm64/llmware/libgraph_llmware.so b/llmware/lib/darwin/arm64/llmware/libgraph_llmware.so new file mode 100755 index 0000000..ae98bef Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/libgraph_llmware.so differ diff --git a/llmware/lib/darwin/arm64/llmware/liboffice_llmware.so b/llmware/lib/darwin/arm64/llmware/liboffice_llmware.so new file mode 100755 index 0000000..fb19d6a Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/liboffice_llmware.so differ diff --git a/llmware/lib/darwin/arm64/llmware/libpdf_llmware.so b/llmware/lib/darwin/arm64/llmware/libpdf_llmware.so new file mode 100755 index 0000000..7d1de47 Binary files /dev/null and b/llmware/lib/darwin/arm64/llmware/libpdf_llmware.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-base.so b/llmware/lib/gguf/gguf_dgx/libggml-base.so new file mode 100644 index 0000000..6f87f73 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-base.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-base.so.0 b/llmware/lib/gguf/gguf_dgx/libggml-base.so.0 new file mode 100644 index 0000000..6f87f73 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-base.so.0 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-base.so.0.9.5 b/llmware/lib/gguf/gguf_dgx/libggml-base.so.0.9.5 new file mode 100644 index 0000000..6f87f73 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-base.so.0.9.5 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-cpu.so b/llmware/lib/gguf/gguf_dgx/libggml-cpu.so new file mode 100644 index 0000000..baa808b Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-cpu.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-cpu.so.0 b/llmware/lib/gguf/gguf_dgx/libggml-cpu.so.0 new file mode 100644 index 0000000..baa808b Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-cpu.so.0 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-cpu.so.0.9.5 b/llmware/lib/gguf/gguf_dgx/libggml-cpu.so.0.9.5 new file mode 100644 index 0000000..baa808b Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-cpu.so.0.9.5 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-cuda.so b/llmware/lib/gguf/gguf_dgx/libggml-cuda.so new file mode 100644 index 0000000..a31cc14 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-cuda.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-cuda.so.0 b/llmware/lib/gguf/gguf_dgx/libggml-cuda.so.0 new file mode 100644 index 0000000..a31cc14 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-cuda.so.0 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml-cuda.so.0.9.5 b/llmware/lib/gguf/gguf_dgx/libggml-cuda.so.0.9.5 new file mode 100644 index 0000000..a31cc14 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml-cuda.so.0.9.5 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml.so b/llmware/lib/gguf/gguf_dgx/libggml.so new file mode 100644 index 0000000..13afaea Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml.so.0 b/llmware/lib/gguf/gguf_dgx/libggml.so.0 new file mode 100644 index 0000000..13afaea Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml.so.0 differ diff --git a/llmware/lib/gguf/gguf_dgx/libggml.so.0.9.5 b/llmware/lib/gguf/gguf_dgx/libggml.so.0.9.5 new file mode 100644 index 0000000..13afaea Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libggml.so.0.9.5 differ diff --git a/llmware/lib/gguf/gguf_dgx/libllama.so b/llmware/lib/gguf/gguf_dgx/libllama.so new file mode 100644 index 0000000..191ae71 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libllama.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libllama.so.0 b/llmware/lib/gguf/gguf_dgx/libllama.so.0 new file mode 100644 index 0000000..191ae71 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libllama.so.0 differ diff --git a/llmware/lib/gguf/gguf_dgx/libllama.so.0.0.7819 b/llmware/lib/gguf/gguf_dgx/libllama.so.0.0.7819 new file mode 100644 index 0000000..191ae71 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libllama.so.0.0.7819 differ diff --git a/llmware/lib/gguf/gguf_dgx/libmtmd.so b/llmware/lib/gguf/gguf_dgx/libmtmd.so new file mode 100644 index 0000000..5678971 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libmtmd.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libmtmd.so.0 b/llmware/lib/gguf/gguf_dgx/libmtmd.so.0 new file mode 100644 index 0000000..5678971 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libmtmd.so.0 differ diff --git a/llmware/lib/gguf/gguf_dgx/libmtmd.so.0.0.7819 b/llmware/lib/gguf/gguf_dgx/libmtmd.so.0.0.7819 new file mode 100644 index 0000000..5678971 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libmtmd.so.0.0.7819 differ diff --git a/llmware/lib/gguf/gguf_dgx/libwhisper.so b/llmware/lib/gguf/gguf_dgx/libwhisper.so new file mode 100755 index 0000000..f8abb05 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libwhisper.so differ diff --git a/llmware/lib/gguf/gguf_dgx/libwhisper.so.1 b/llmware/lib/gguf/gguf_dgx/libwhisper.so.1 new file mode 100755 index 0000000..f8abb05 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libwhisper.so.1 differ diff --git a/llmware/lib/gguf/gguf_dgx/libwhisper.so.1.8.3 b/llmware/lib/gguf/gguf_dgx/libwhisper.so.1.8.3 new file mode 100755 index 0000000..f8abb05 Binary files /dev/null and b/llmware/lib/gguf/gguf_dgx/libwhisper.so.1.8.3 differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libggml-base.so b/llmware/lib/gguf/gguf_linux_cuda/libggml-base.so new file mode 100755 index 0000000..76793e0 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libggml-base.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libggml-cpu.so b/llmware/lib/gguf/gguf_linux_cuda/libggml-cpu.so new file mode 100755 index 0000000..df40868 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libggml-cpu.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libggml-cuda.so b/llmware/lib/gguf/gguf_linux_cuda/libggml-cuda.so new file mode 100755 index 0000000..7ed7240 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libggml-cuda.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libggml.so b/llmware/lib/gguf/gguf_linux_cuda/libggml.so new file mode 100755 index 0000000..96e1c65 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libggml.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libgomp.so.1 b/llmware/lib/gguf/gguf_linux_cuda/libgomp.so.1 new file mode 100644 index 0000000..73a97c0 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libgomp.so.1 differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libgomp.so.1.0.0 b/llmware/lib/gguf/gguf_linux_cuda/libgomp.so.1.0.0 new file mode 100644 index 0000000..73a97c0 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libgomp.so.1.0.0 differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libllama.so b/llmware/lib/gguf/gguf_linux_cuda/libllama.so new file mode 100755 index 0000000..98624e6 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libllama.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libmtmd.so b/llmware/lib/gguf/gguf_linux_cuda/libmtmd.so new file mode 100644 index 0000000..b267d01 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libmtmd.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so b/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so new file mode 100755 index 0000000..fe2706b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so.1 b/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so.1 new file mode 100755 index 0000000..fe2706b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so.1 differ diff --git a/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so.1.8.3 b/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so.1.8.3 new file mode 100755 index 0000000..fe2706b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_cuda/libwhisper.so.1.8.3 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml-base.so b/llmware/lib/gguf/gguf_linux_x86/libggml-base.so new file mode 100644 index 0000000..3cd9838 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml-base.so differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml-base.so.0 b/llmware/lib/gguf/gguf_linux_x86/libggml-base.so.0 new file mode 100644 index 0000000..3cd9838 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml-base.so.0 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml-base.so.0.9.8 b/llmware/lib/gguf/gguf_linux_x86/libggml-base.so.0.9.8 new file mode 100644 index 0000000..3cd9838 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml-base.so.0.9.8 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so b/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so new file mode 100644 index 0000000..45d151e Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so.0 b/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so.0 new file mode 100644 index 0000000..45d151e Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so.0 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so.0.9.8 b/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so.0.9.8 new file mode 100644 index 0000000..45d151e Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml-cpu.so.0.9.8 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml.so b/llmware/lib/gguf/gguf_linux_x86/libggml.so new file mode 100644 index 0000000..3a05552 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml.so differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml.so.0 b/llmware/lib/gguf/gguf_linux_x86/libggml.so.0 new file mode 100644 index 0000000..3a05552 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml.so.0 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libggml.so.0.9.8 b/llmware/lib/gguf/gguf_linux_x86/libggml.so.0.9.8 new file mode 100644 index 0000000..3a05552 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libggml.so.0.9.8 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libgomp.so.1 b/llmware/lib/gguf/gguf_linux_x86/libgomp.so.1 new file mode 100644 index 0000000..73a97c0 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libgomp.so.1 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libgomp.so.1.0.0 b/llmware/lib/gguf/gguf_linux_x86/libgomp.so.1.0.0 new file mode 100644 index 0000000..73a97c0 Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libgomp.so.1.0.0 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libllama.so b/llmware/lib/gguf/gguf_linux_x86/libllama.so new file mode 100644 index 0000000..8dc861d Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libllama.so differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libllama.so.0 b/llmware/lib/gguf/gguf_linux_x86/libllama.so.0 new file mode 100644 index 0000000..8dc861d Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libllama.so.0 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libllama.so.0.0.8538 b/llmware/lib/gguf/gguf_linux_x86/libllama.so.0.0.8538 new file mode 100644 index 0000000..8dc861d Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libllama.so.0.0.8538 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libmtmd.so b/llmware/lib/gguf/gguf_linux_x86/libmtmd.so new file mode 100644 index 0000000..a750d0b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libmtmd.so differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libmtmd.so.0 b/llmware/lib/gguf/gguf_linux_x86/libmtmd.so.0 new file mode 100644 index 0000000..a750d0b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libmtmd.so.0 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libmtmd.so.0.0.8538 b/llmware/lib/gguf/gguf_linux_x86/libmtmd.so.0.0.8538 new file mode 100644 index 0000000..a750d0b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libmtmd.so.0.0.8538 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libwhisper.so b/llmware/lib/gguf/gguf_linux_x86/libwhisper.so new file mode 100755 index 0000000..fe2706b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libwhisper.so differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libwhisper.so.1 b/llmware/lib/gguf/gguf_linux_x86/libwhisper.so.1 new file mode 100755 index 0000000..fe2706b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libwhisper.so.1 differ diff --git a/llmware/lib/gguf/gguf_linux_x86/libwhisper.so.1.8.3 b/llmware/lib/gguf/gguf_linux_x86/libwhisper.so.1.8.3 new file mode 100755 index 0000000..fe2706b Binary files /dev/null and b/llmware/lib/gguf/gguf_linux_x86/libwhisper.so.1.8.3 differ diff --git a/llmware/lib/gguf/gguf_mac/libggml-base.0.10.2.dylib b/llmware/lib/gguf/gguf_mac/libggml-base.0.10.2.dylib new file mode 100755 index 0000000..443d86c Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libggml-base.0.10.2.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libggml-base.0.dylib b/llmware/lib/gguf/gguf_mac/libggml-base.0.dylib new file mode 120000 index 0000000..9cc2230 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-base.0.dylib @@ -0,0 +1 @@ +libggml-base.0.10.2.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-base.dylib b/llmware/lib/gguf/gguf_mac/libggml-base.dylib new file mode 120000 index 0000000..f08d1fc --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-base.dylib @@ -0,0 +1 @@ +libggml-base.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-blas.0.10.2.dylib b/llmware/lib/gguf/gguf_mac/libggml-blas.0.10.2.dylib new file mode 100755 index 0000000..496fe9f Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libggml-blas.0.10.2.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libggml-blas.0.dylib b/llmware/lib/gguf/gguf_mac/libggml-blas.0.dylib new file mode 120000 index 0000000..6fc0196 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-blas.0.dylib @@ -0,0 +1 @@ +libggml-blas.0.10.2.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-blas.dylib b/llmware/lib/gguf/gguf_mac/libggml-blas.dylib new file mode 120000 index 0000000..28748a5 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-blas.dylib @@ -0,0 +1 @@ +libggml-blas.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-cpu.0.10.2.dylib b/llmware/lib/gguf/gguf_mac/libggml-cpu.0.10.2.dylib new file mode 100755 index 0000000..549b8d0 Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libggml-cpu.0.10.2.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libggml-cpu.0.dylib b/llmware/lib/gguf/gguf_mac/libggml-cpu.0.dylib new file mode 120000 index 0000000..e034146 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-cpu.0.dylib @@ -0,0 +1 @@ +libggml-cpu.0.10.2.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-cpu.dylib b/llmware/lib/gguf/gguf_mac/libggml-cpu.dylib new file mode 120000 index 0000000..ac40938 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-cpu.dylib @@ -0,0 +1 @@ +libggml-cpu.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-metal.0.10.2.dylib b/llmware/lib/gguf/gguf_mac/libggml-metal.0.10.2.dylib new file mode 100755 index 0000000..f9fd1a4 Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libggml-metal.0.10.2.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libggml-metal.0.dylib b/llmware/lib/gguf/gguf_mac/libggml-metal.0.dylib new file mode 120000 index 0000000..b7b1e92 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-metal.0.dylib @@ -0,0 +1 @@ +libggml-metal.0.10.2.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml-metal.dylib b/llmware/lib/gguf/gguf_mac/libggml-metal.dylib new file mode 120000 index 0000000..8210fd7 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml-metal.dylib @@ -0,0 +1 @@ +libggml-metal.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml.0.10.2.dylib b/llmware/lib/gguf/gguf_mac/libggml.0.10.2.dylib new file mode 100755 index 0000000..3329c9b Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libggml.0.10.2.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libggml.0.dylib b/llmware/lib/gguf/gguf_mac/libggml.0.dylib new file mode 120000 index 0000000..facac1e --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml.0.dylib @@ -0,0 +1 @@ +libggml.0.10.2.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libggml.dylib b/llmware/lib/gguf/gguf_mac/libggml.dylib new file mode 120000 index 0000000..0ef7111 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libggml.dylib @@ -0,0 +1 @@ +libggml.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libllama-common.0.0.9021.dylib b/llmware/lib/gguf/gguf_mac/libllama-common.0.0.9021.dylib new file mode 100755 index 0000000..0475849 Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libllama-common.0.0.9021.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libllama-common.0.dylib b/llmware/lib/gguf/gguf_mac/libllama-common.0.dylib new file mode 120000 index 0000000..2a7bc73 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libllama-common.0.dylib @@ -0,0 +1 @@ +libllama-common.0.0.9021.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libllama-common.dylib b/llmware/lib/gguf/gguf_mac/libllama-common.dylib new file mode 120000 index 0000000..2cb98c3 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libllama-common.dylib @@ -0,0 +1 @@ +libllama-common.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libllama.0.0.9021.dylib b/llmware/lib/gguf/gguf_mac/libllama.0.0.9021.dylib new file mode 100755 index 0000000..d4e13b7 Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libllama.0.0.9021.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libllama.0.dylib b/llmware/lib/gguf/gguf_mac/libllama.0.dylib new file mode 120000 index 0000000..6785b54 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libllama.0.dylib @@ -0,0 +1 @@ +libllama.0.0.9021.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libllama.dylib b/llmware/lib/gguf/gguf_mac/libllama.dylib new file mode 120000 index 0000000..d76d521 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libllama.dylib @@ -0,0 +1 @@ +libllama.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libmtmd.0.0.9021.dylib b/llmware/lib/gguf/gguf_mac/libmtmd.0.0.9021.dylib new file mode 100755 index 0000000..c15bdba Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libmtmd.0.0.9021.dylib differ diff --git a/llmware/lib/gguf/gguf_mac/libmtmd.0.dylib b/llmware/lib/gguf/gguf_mac/libmtmd.0.dylib new file mode 120000 index 0000000..7cb15aa --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libmtmd.0.dylib @@ -0,0 +1 @@ +libmtmd.0.0.9021.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libmtmd.dylib b/llmware/lib/gguf/gguf_mac/libmtmd.dylib new file mode 120000 index 0000000..21fd179 --- /dev/null +++ b/llmware/lib/gguf/gguf_mac/libmtmd.dylib @@ -0,0 +1 @@ +libmtmd.0.dylib \ No newline at end of file diff --git a/llmware/lib/gguf/gguf_mac/libwhisper_mac_metal_155.dylib b/llmware/lib/gguf/gguf_mac/libwhisper_mac_metal_155.dylib new file mode 100755 index 0000000..2bed0f0 Binary files /dev/null and b/llmware/lib/gguf/gguf_mac/libwhisper_mac_metal_155.dylib differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-base.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-base.dll new file mode 100644 index 0000000..4d2cc5b Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-base.dll differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-cpu.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-cpu.dll new file mode 100644 index 0000000..cfdc58b Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-cpu.dll differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-vulkan.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-vulkan.dll new file mode 100644 index 0000000..2798932 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml-vulkan.dll differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/ggml.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml.dll new file mode 100644 index 0000000..1bfe09a Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/ggml.dll differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/llama.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/llama.dll new file mode 100644 index 0000000..41473de Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/llama.dll differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/mtmd.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/mtmd.dll new file mode 100644 index 0000000..8215417 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/mtmd.dll differ diff --git a/llmware/lib/gguf/gguf_win_amd_vulkan/whisper.dll b/llmware/lib/gguf/gguf_win_amd_vulkan/whisper.dll new file mode 100644 index 0000000..bde7323 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_amd_vulkan/whisper.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/ggml-base.dll b/llmware/lib/gguf/gguf_win_arm64/ggml-base.dll new file mode 100644 index 0000000..7d4b18c Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/ggml-base.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/ggml-cpu.dll b/llmware/lib/gguf/gguf_win_arm64/ggml-cpu.dll new file mode 100644 index 0000000..50bf4c6 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/ggml-cpu.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/ggml.dll b/llmware/lib/gguf/gguf_win_arm64/ggml.dll new file mode 100644 index 0000000..153c638 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/ggml.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/llama-common.dll b/llmware/lib/gguf/gguf_win_arm64/llama-common.dll new file mode 100644 index 0000000..30cb567 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/llama-common.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/llama.dll b/llmware/lib/gguf/gguf_win_arm64/llama.dll new file mode 100644 index 0000000..efbdbd8 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/llama.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/mtmd.dll b/llmware/lib/gguf/gguf_win_arm64/mtmd.dll new file mode 100644 index 0000000..20071a0 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/mtmd.dll differ diff --git a/llmware/lib/gguf/gguf_win_arm64/whisper.dll b/llmware/lib/gguf/gguf_win_arm64/whisper.dll new file mode 100644 index 0000000..87cc087 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_arm64/whisper.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/ggml-base.dll b/llmware/lib/gguf/gguf_win_cuda/ggml-base.dll new file mode 100644 index 0000000..9cb8c1c Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/ggml-base.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/ggml-cpu.dll b/llmware/lib/gguf/gguf_win_cuda/ggml-cpu.dll new file mode 100644 index 0000000..6c574e9 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/ggml-cpu.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/ggml-cuda.dll b/llmware/lib/gguf/gguf_win_cuda/ggml-cuda.dll new file mode 100644 index 0000000..8a1e7da Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/ggml-cuda.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/ggml.dll b/llmware/lib/gguf/gguf_win_cuda/ggml.dll new file mode 100644 index 0000000..e6c1fe4 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/ggml.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/llama.dll b/llmware/lib/gguf/gguf_win_cuda/llama.dll new file mode 100644 index 0000000..9bee856 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/llama.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/mtmd.dll b/llmware/lib/gguf/gguf_win_cuda/mtmd.dll new file mode 100644 index 0000000..7ea1d08 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/mtmd.dll differ diff --git a/llmware/lib/gguf/gguf_win_cuda/whisper.dll b/llmware/lib/gguf/gguf_win_cuda/whisper.dll new file mode 100644 index 0000000..bde7323 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_cuda/whisper.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/ggml-base.dll b/llmware/lib/gguf/gguf_win_x86/ggml-base.dll new file mode 100644 index 0000000..36b372d Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/ggml-base.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/ggml-cpu.dll b/llmware/lib/gguf/gguf_win_x86/ggml-cpu.dll new file mode 100644 index 0000000..64179dd Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/ggml-cpu.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/ggml.dll b/llmware/lib/gguf/gguf_win_x86/ggml.dll new file mode 100644 index 0000000..82311de Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/ggml.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/llama-common.dll b/llmware/lib/gguf/gguf_win_x86/llama-common.dll new file mode 100644 index 0000000..effb5ed Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/llama-common.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/llama.dll b/llmware/lib/gguf/gguf_win_x86/llama.dll new file mode 100644 index 0000000..2483060 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/llama.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/mtmd.dll b/llmware/lib/gguf/gguf_win_x86/mtmd.dll new file mode 100644 index 0000000..c8bf11a Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/mtmd.dll differ diff --git a/llmware/lib/gguf/gguf_win_x86/whisper.dll b/llmware/lib/gguf/gguf_win_x86/whisper.dll new file mode 100644 index 0000000..bde7323 Binary files /dev/null and b/llmware/lib/gguf/gguf_win_x86/whisper.dll differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libbson-1-e2fbf069.0.so.0.0.0 b/llmware/lib/linux/aarch64/llmware.libs/libbson-1-e2fbf069.0.so.0.0.0 new file mode 100644 index 0000000..680f285 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libbson-1-e2fbf069.0.so.0.0.0 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libcom_err-057ba42b.so.2.1 b/llmware/lib/linux/aarch64/llmware.libs/libcom_err-057ba42b.so.2.1 new file mode 100644 index 0000000..19156c8 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libcom_err-057ba42b.so.2.1 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libcrypto-13880bfc.so.1.0.2k b/llmware/lib/linux/aarch64/llmware.libs/libcrypto-13880bfc.so.1.0.2k new file mode 100644 index 0000000..be9e476 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libcrypto-13880bfc.so.1.0.2k differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libgssapi_krb5-fec99a71.so.2.2 b/llmware/lib/linux/aarch64/llmware.libs/libgssapi_krb5-fec99a71.so.2.2 new file mode 100644 index 0000000..3229fa5 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libgssapi_krb5-fec99a71.so.2.2 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libk5crypto-47ac5e52.so.3.1 b/llmware/lib/linux/aarch64/llmware.libs/libk5crypto-47ac5e52.so.3.1 new file mode 100644 index 0000000..990bf32 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libk5crypto-47ac5e52.so.3.1 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libkeyutils-19c64d08.so.1.5 b/llmware/lib/linux/aarch64/llmware.libs/libkeyutils-19c64d08.so.1.5 new file mode 100644 index 0000000..dc15bc5 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libkeyutils-19c64d08.so.1.5 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libkrb5-90a0ef7c.so.3.3 b/llmware/lib/linux/aarch64/llmware.libs/libkrb5-90a0ef7c.so.3.3 new file mode 100644 index 0000000..4d8705b Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libkrb5-90a0ef7c.so.3.3 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libkrb5support-73f3f43d.so.0.1 b/llmware/lib/linux/aarch64/llmware.libs/libkrb5support-73f3f43d.so.0.1 new file mode 100644 index 0000000..da799cd Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libkrb5support-73f3f43d.so.0.1 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/liblzma-6bd50f17.so.5.2.2 b/llmware/lib/linux/aarch64/llmware.libs/liblzma-6bd50f17.so.5.2.2 new file mode 100644 index 0000000..41bbccd Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/liblzma-6bd50f17.so.5.2.2 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libmongoc-1-5336003e.0.so.0.0.0 b/llmware/lib/linux/aarch64/llmware.libs/libmongoc-1-5336003e.0.so.0.0.0 new file mode 100644 index 0000000..46a5cfa Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libmongoc-1-5336003e.0.so.0.0.0 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libpcre-6b975b27.so.1.2.0 b/llmware/lib/linux/aarch64/llmware.libs/libpcre-6b975b27.so.1.2.0 new file mode 100644 index 0000000..0409d80 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libpcre-6b975b27.so.1.2.0 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libpng16-7287d4e1.so.16.40.0 b/llmware/lib/linux/aarch64/llmware.libs/libpng16-7287d4e1.so.16.40.0 new file mode 100644 index 0000000..39ae854 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libpng16-7287d4e1.so.16.40.0 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libpq-954018e6.so.5.16 b/llmware/lib/linux/aarch64/llmware.libs/libpq-954018e6.so.5.16 new file mode 100644 index 0000000..ad92056 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libpq-954018e6.so.5.16 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libselinux-ae24b712.so.1 b/llmware/lib/linux/aarch64/llmware.libs/libselinux-ae24b712.so.1 new file mode 100644 index 0000000..da3ef42 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libselinux-ae24b712.so.1 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libsqlite3-0245f74e.so.0.8.6 b/llmware/lib/linux/aarch64/llmware.libs/libsqlite3-0245f74e.so.0.8.6 new file mode 100644 index 0000000..8faf930 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libsqlite3-0245f74e.so.0.8.6 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libssl-e50c59ab.so.1.0.2k b/llmware/lib/linux/aarch64/llmware.libs/libssl-e50c59ab.so.1.0.2k new file mode 100644 index 0000000..c18cfbb Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libssl-e50c59ab.so.1.0.2k differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libxml2-06c1d338.so.2.9.1 b/llmware/lib/linux/aarch64/llmware.libs/libxml2-06c1d338.so.2.9.1 new file mode 100644 index 0000000..18435c6 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libxml2-06c1d338.so.2.9.1 differ diff --git a/llmware/lib/linux/aarch64/llmware.libs/libzip-ca517e33.so.5.5 b/llmware/lib/linux/aarch64/llmware.libs/libzip-ca517e33.so.5.5 new file mode 100644 index 0000000..820b7a0 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware.libs/libzip-ca517e33.so.5.5 differ diff --git a/llmware/lib/linux/aarch64/llmware/libgraph_llmware.so b/llmware/lib/linux/aarch64/llmware/libgraph_llmware.so new file mode 100644 index 0000000..40a1627 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware/libgraph_llmware.so differ diff --git a/llmware/lib/linux/aarch64/llmware/liboffice_llmware.so b/llmware/lib/linux/aarch64/llmware/liboffice_llmware.so new file mode 100644 index 0000000..96afb34 Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware/liboffice_llmware.so differ diff --git a/llmware/lib/linux/aarch64/llmware/libpdf_llmware.so b/llmware/lib/linux/aarch64/llmware/libpdf_llmware.so new file mode 100644 index 0000000..071335e Binary files /dev/null and b/llmware/lib/linux/aarch64/llmware/libpdf_llmware.so differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libbson-1-5dd9c7b2.0.so.0.0.0 b/llmware/lib/linux/x86_64/llmware.libs/libbson-1-5dd9c7b2.0.so.0.0.0 new file mode 100644 index 0000000..94c8256 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libbson-1-5dd9c7b2.0.so.0.0.0 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libbz2-4488544d.so.1.0.4 b/llmware/lib/linux/x86_64/llmware.libs/libbz2-4488544d.so.1.0.4 new file mode 100644 index 0000000..abb73f2 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libbz2-4488544d.so.1.0.4 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libcrypto-d7a32359.so.1.1 b/llmware/lib/linux/x86_64/llmware.libs/libcrypto-d7a32359.so.1.1 new file mode 100644 index 0000000..4596a99 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libcrypto-d7a32359.so.1.1 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libmongoc-1-0761b075.0.so.0.0.0 b/llmware/lib/linux/x86_64/llmware.libs/libmongoc-1-0761b075.0.so.0.0.0 new file mode 100644 index 0000000..0c1494c Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libmongoc-1-0761b075.0.so.0.0.0 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libpng16-7453c768.so.16.40.0 b/llmware/lib/linux/x86_64/llmware.libs/libpng16-7453c768.so.16.40.0 new file mode 100755 index 0000000..217a6ef Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libpng16-7453c768.so.16.40.0 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libpq-35bcdb96.so.5.16 b/llmware/lib/linux/x86_64/llmware.libs/libpq-35bcdb96.so.5.16 new file mode 100755 index 0000000..362947b Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libpq-35bcdb96.so.5.16 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libsnappy-e38178cf.so.1.1.8 b/llmware/lib/linux/x86_64/llmware.libs/libsnappy-e38178cf.so.1.1.8 new file mode 100644 index 0000000..4411915 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libsnappy-e38178cf.so.1.1.8 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libsqlite3-3cd414b8.so.0.8.6 b/llmware/lib/linux/x86_64/llmware.libs/libsqlite3-3cd414b8.so.0.8.6 new file mode 100755 index 0000000..237cf1d Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libsqlite3-3cd414b8.so.0.8.6 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libssl-6e513d0a.so.1.1 b/llmware/lib/linux/x86_64/llmware.libs/libssl-6e513d0a.so.1.1 new file mode 100644 index 0000000..800c1e4 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libssl-6e513d0a.so.1.1 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libxml2-726323b3.so.2.13.0 b/llmware/lib/linux/x86_64/llmware.libs/libxml2-726323b3.so.2.13.0 new file mode 100755 index 0000000..5572168 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libxml2-726323b3.so.2.13.0 differ diff --git a/llmware/lib/linux/x86_64/llmware.libs/libzip-9c23ab55.so.5.0 b/llmware/lib/linux/x86_64/llmware.libs/libzip-9c23ab55.so.5.0 new file mode 100644 index 0000000..67f5831 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware.libs/libzip-9c23ab55.so.5.0 differ diff --git a/llmware/lib/linux/x86_64/llmware/libgraph_llmware.so b/llmware/lib/linux/x86_64/llmware/libgraph_llmware.so new file mode 100755 index 0000000..cff7333 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware/libgraph_llmware.so differ diff --git a/llmware/lib/linux/x86_64/llmware/liboffice_llmware.so b/llmware/lib/linux/x86_64/llmware/liboffice_llmware.so new file mode 100755 index 0000000..a0a7f43 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware/liboffice_llmware.so differ diff --git a/llmware/lib/linux/x86_64/llmware/libpdf_llmware.so b/llmware/lib/linux/x86_64/llmware/libpdf_llmware.so new file mode 100755 index 0000000..f8f4563 Binary files /dev/null and b/llmware/lib/linux/x86_64/llmware/libpdf_llmware.so differ diff --git a/llmware/lib/windows/arm64/llmware/libbz2-1.dll b/llmware/lib/windows/arm64/llmware/libbz2-1.dll new file mode 100644 index 0000000..bfd00d6 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libbz2-1.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libcrypto-3.dll b/llmware/lib/windows/arm64/llmware/libcrypto-3.dll new file mode 100644 index 0000000..21428af Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libcrypto-3.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libiconv-2.dll b/llmware/lib/windows/arm64/llmware/libiconv-2.dll new file mode 100644 index 0000000..02ad8d0 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libiconv-2.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libintl-8.dll b/llmware/lib/windows/arm64/llmware/libintl-8.dll new file mode 100644 index 0000000..d2bc922 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libintl-8.dll differ diff --git a/llmware/lib/windows/arm64/llmware/liblzma-5.dll b/llmware/lib/windows/arm64/llmware/liblzma-5.dll new file mode 100644 index 0000000..341c8cd Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/liblzma-5.dll differ diff --git a/llmware/lib/windows/arm64/llmware/liboffice_llmware.dll b/llmware/lib/windows/arm64/llmware/liboffice_llmware.dll new file mode 100644 index 0000000..4bae4b9 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/liboffice_llmware.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libpdf_llmware.dll b/llmware/lib/windows/arm64/llmware/libpdf_llmware.dll new file mode 100644 index 0000000..cdb5496 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libpdf_llmware.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libpng16-16.dll b/llmware/lib/windows/arm64/llmware/libpng16-16.dll new file mode 100644 index 0000000..266399f Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libpng16-16.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libpq.dll b/llmware/lib/windows/arm64/llmware/libpq.dll new file mode 100644 index 0000000..f7a51f2 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libpq.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libsqlite3-0.dll b/llmware/lib/windows/arm64/llmware/libsqlite3-0.dll new file mode 100644 index 0000000..183ed20 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libsqlite3-0.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libssl-3.dll b/llmware/lib/windows/arm64/llmware/libssl-3.dll new file mode 100644 index 0000000..b1fe4e6 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libssl-3.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libxml2-2.dll b/llmware/lib/windows/arm64/llmware/libxml2-2.dll new file mode 100644 index 0000000..f640eca Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libxml2-2.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libzip.dll b/llmware/lib/windows/arm64/llmware/libzip.dll new file mode 100644 index 0000000..18b2df5 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libzip.dll differ diff --git a/llmware/lib/windows/arm64/llmware/libzstd.dll b/llmware/lib/windows/arm64/llmware/libzstd.dll new file mode 100644 index 0000000..f31c090 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/libzstd.dll differ diff --git a/llmware/lib/windows/arm64/llmware/zlib1.dll b/llmware/lib/windows/arm64/llmware/zlib1.dll new file mode 100644 index 0000000..41eb777 Binary files /dev/null and b/llmware/lib/windows/arm64/llmware/zlib1.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libbson-1.0.dll b/llmware/lib/windows/x86_64/llmware/libbson-1.0.dll new file mode 100644 index 0000000..d58e7c2 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libbson-1.0.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libbz2-1.dll b/llmware/lib/windows/x86_64/llmware/libbz2-1.dll new file mode 100644 index 0000000..83d25c4 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libbz2-1.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libcrypto-3-x64.dll b/llmware/lib/windows/x86_64/llmware/libcrypto-3-x64.dll new file mode 100644 index 0000000..67d8ca1 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libcrypto-3-x64.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libgcc_s_seh-1.dll b/llmware/lib/windows/x86_64/llmware/libgcc_s_seh-1.dll new file mode 100644 index 0000000..b587b37 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libgcc_s_seh-1.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libgraph_llmware.dll b/llmware/lib/windows/x86_64/llmware/libgraph_llmware.dll new file mode 100644 index 0000000..0c87b77 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libgraph_llmware.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libiconv-2.dll b/llmware/lib/windows/x86_64/llmware/libiconv-2.dll new file mode 100644 index 0000000..d6a8284 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libiconv-2.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libintl-8.dll b/llmware/lib/windows/x86_64/llmware/libintl-8.dll new file mode 100644 index 0000000..b7ca4f2 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libintl-8.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/liblzma-5.dll b/llmware/lib/windows/x86_64/llmware/liblzma-5.dll new file mode 100644 index 0000000..187428a Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/liblzma-5.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libmongoc-1.0.dll b/llmware/lib/windows/x86_64/llmware/libmongoc-1.0.dll new file mode 100644 index 0000000..d99a9f7 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libmongoc-1.0.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/liboffice_llmware.dll b/llmware/lib/windows/x86_64/llmware/liboffice_llmware.dll new file mode 100644 index 0000000..27be867 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/liboffice_llmware.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libpdf_llmware.dll b/llmware/lib/windows/x86_64/llmware/libpdf_llmware.dll new file mode 100644 index 0000000..64d5b06 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libpdf_llmware.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libpng16-16.dll b/llmware/lib/windows/x86_64/llmware/libpng16-16.dll new file mode 100644 index 0000000..392f62a Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libpng16-16.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libpq.dll b/llmware/lib/windows/x86_64/llmware/libpq.dll new file mode 100644 index 0000000..650bf59 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libpq.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libsqlite3-0.dll b/llmware/lib/windows/x86_64/llmware/libsqlite3-0.dll new file mode 100644 index 0000000..227147c Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libsqlite3-0.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libssl-3-x64.dll b/llmware/lib/windows/x86_64/llmware/libssl-3-x64.dll new file mode 100644 index 0000000..34ce55d Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libssl-3-x64.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libtiff-6.dll b/llmware/lib/windows/x86_64/llmware/libtiff-6.dll new file mode 100644 index 0000000..ffbfe6d Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libtiff-6.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libwinpthread-1.dll b/llmware/lib/windows/x86_64/llmware/libwinpthread-1.dll new file mode 100644 index 0000000..5f4efbd Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libwinpthread-1.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libxml2-2.dll b/llmware/lib/windows/x86_64/llmware/libxml2-2.dll new file mode 100644 index 0000000..a632fb5 Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libxml2-2.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libzip.dll b/llmware/lib/windows/x86_64/llmware/libzip.dll new file mode 100644 index 0000000..15fcf7e Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libzip.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/libzstd.dll b/llmware/lib/windows/x86_64/llmware/libzstd.dll new file mode 100644 index 0000000..49e0bee Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/libzstd.dll differ diff --git a/llmware/lib/windows/x86_64/llmware/zlib1.dll b/llmware/lib/windows/x86_64/llmware/zlib1.dll new file mode 100644 index 0000000..0032dbe Binary files /dev/null and b/llmware/lib/windows/x86_64/llmware/zlib1.dll differ diff --git a/llmware/library.py b/llmware/library.py new file mode 100755 index 0000000..97c314a --- /dev/null +++ b/llmware/library.py @@ -0,0 +1,1436 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The library module implements the logic for managing unstructured information (the text). + +The module implements the two classes Library and LibraryCatalog. Library is responsible for organizing a +collection of text and is the interface for the Parser and Embedding classes. In addition, the Library object +is passed to the Query and Prompt objects. The Library class uses the LibraryCatalog for creating, deleting, +updating, and other tasks pertaining to Libraries via the Library Card. +""" + +import shutil +import os +import json +import logging + +from llmware.configs import LLMWareConfig, LLMWareTableSchema, LLMWareException +from llmware.util import Utilities +from llmware.parsers import Parser +from llmware.models import ModelCatalog +from llmware.resources import CollectionRetrieval, CollectionWriter, CloudBucketManager +from llmware.embeddings import EmbeddingHandler + +logger = logging.getLogger(__name__) + + +class Library: + + """Implements the interface to manage a collection of unstructured information as a ``Library``, i.e. a + library is an indexed collection of texts, tables and images extracted from parsed files. + + Returns + ------- + library : Library + A new ``Library`` object. + """ + + def __init__(self): + + # default settings for basic parameters + self.account_name = None + self.library_name = None + + # base file paths in each library + self.library_main_path = None + + # each of these paths hang off library_main_path + self.file_copy_path = None + self.image_path = None + self.dataset_path = None + self.nlp_path = None + self.output_path = None + self.tmp_path = None + self.embedding_path = None + + # default key structure of block -> re-order for nicer display + self.default_keys = ["block_ID", "doc_ID", "content_type", "file_type","master_index","master_index2", + "coords_x", "coords_y", "coords_cx", "coords_cy", "author_or_speaker", "modified_date", + "created_date", "creator_tool", "added_to_collection", "file_source", + # changing to 'table_block' and 'text_block' + "table_block", "external_files", "text_block", "header_text", "text_search", + "user_tags", "special_field1", "special_field2", "special_field3","graph_status","dialog", + "embedding_flags"] + + self.library_block_schema = LLMWareTableSchema().get_block_schema() + + # default library card elements + self.default_library_card = ["library_name", "embedding_status", "embedding_model", "embedding_db", + "embedded_blocks", "embedding_dims", "time_stamp", + "knowledge_graph", "unique_doc_id", "documents", "blocks", "images", "pages", + "tables"] + + self.block_size_target_characters = 400 + + # attributes used in parsing workflow + self.doc_ID = 0 + self.block_ID = 0 + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + # add checks for /tmp and /accounts in place + library_path = LLMWareConfig.get_library_path() + tmp_path = LLMWareConfig.get_tmp_path() + + if not os.path.exists(library_path): + os.mkdir(library_path) + os.chmod(library_path, 0o777) + + if not os.path.exists(tmp_path): + os.mkdir(tmp_path) + os.chmod(tmp_path, 0o777) + + # explicit constructor to create a new library + def create_new_library(self, library_name, account_name="llmware"): + """Explicit constructor to create a new library with selected name. + + If a library with the same name already exists, it will load the existing library. + + Checks if library_name is safe. If not, it will change library_name to a safe name. + + Parameters + ---------- + library_name : str + name of the library to create + + account_name : str, default="llmware" + name of the account associated with the library + + Returns + ------- + library : Library + A new ``Library`` object representing the newly created or loaded existing library + """ + + # note: default behavior - if library with same name already exists, then it loads existing library + + self.library_name = library_name + self.account_name = account_name + + # apply safety check to library_name path + library_name = Utilities().secure_filename(library_name) + + library_exists = self.check_if_library_exists(library_name,account_name) + + if library_exists: + + # do not create + logger.info(f"Library - create_new_library - library already exists - returning library - {library_name} - {account_name}") + + return self.load_library(library_name, account_name) + + # assign self.library_name to the 'safe' library_name + self.library_name = library_name + + # allow 'dynamic' creation of a new account path + account_path = os.path.join(LLMWareConfig.get_library_path(), account_name) + if not os.path.exists(account_path): + os.makedirs(account_path,exist_ok=True) + + # safety check for name based on db + safe_name = CollectionRetrieval(library_name,account_name=self.account_name).safe_name(library_name) + + if safe_name != library_name: + + logger.warning(f"Library - create_new_library - selected library name is being changed for safety on selected resource - " + f"{safe_name}") + + if isinstance(safe_name,str): + library_name = safe_name + self.library_name = safe_name + + else: + raise LLMWareException(message=f"Library - create_new_library - selected name is not " + f"valid library name - {library_name}") + + self.library_main_path = os.path.join(LLMWareConfig.get_library_path(), account_name, library_name) + + # add new file dir for this collection + self.file_copy_path = os.path.join(self.library_main_path,"uploads" + os.sep ) + self.image_path = os.path.join(self.library_main_path, "images" + os.sep) + self.dataset_path = os.path.join(self.library_main_path, "datasets" + os.sep) + self.nlp_path = os.path.join(self.library_main_path, "nlp" + os.sep) + self.output_path = os.path.join(self.library_main_path, "output" + os.sep) + self.tmp_path = os.path.join(self.library_main_path, "tmp" + os.sep) + self.embedding_path = os.path.join(self.library_main_path, "embedding" + os.sep) + + library_folder = os.path.exists(self.library_main_path) + + # this is a new library to create -> build file paths for work products + if not library_folder: + os.mkdir(self.library_main_path) + os.mkdir(self.file_copy_path) + os.mkdir(self.image_path) + os.mkdir(self.dataset_path) + os.mkdir(self.nlp_path) + os.mkdir(self.output_path) + os.mkdir(self.tmp_path) + os.mkdir(self.embedding_path) + os.chmod(self.dataset_path, 0o777) + os.chmod(self.nlp_path, 0o777) + os.chmod(self.output_path, 0o777) + os.chmod(self.tmp_path, 0o777) + os.chmod(self.embedding_path, 0o777) + + new_library_entry = {"library_name": self.library_name, + + # track embedding status - each embedding tracked as new dict in list + # --by default, when library created, no embedding in place + + "embedding": [{"embedding_status": "no", "embedding_model": "none", "embedding_db": "none", + "embedded_blocks":0, "embedding_dims":0, "time_stamp": "NA"}], + + # knowledge graph + # deprecated in 0.5.0 -> will be removed in upcoming releases + "knowledge_graph": "no", + + # doc trackers + "unique_doc_id": 0, "documents": 0, "blocks": 0, "images": 0, "pages": 0, "tables": 0, + + # options to create and set different accounts + "account_name": self.account_name + } + + # LibraryCatalog will register the new library card + new_library_card = LibraryCatalog(self).create_new_library_card(new_library_entry) + + if CollectionWriter(self.library_name,account_name=self.account_name).check_if_table_build_required(): + + CollectionWriter(self.library_name,account_name=self.account_name).create_table(self.library_name, + self.library_block_schema) + + # update collection text index in collection after adding documents + CollectionWriter(self.library_name,account_name=self.account_name).build_text_index() + + return self + + def load_library(self, library_name, account_name="llmware"): + + """Load an existing library by invoking the library string name. + + Parameters + ---------- + library_name : str + Name of the library to load + + account_name : str, default="llmware" + Name of the account associated with the library + + Returns + ------- + library : Library + A new ``Library`` object representing the loaded library + """ + + # first check that library exists + library_exists = self.check_if_library_exists(library_name, account_name=account_name) + + if not library_exists: + raise LLMWareException(message=f"Library - load_library - library not found -" + f"library_name = {library_name} with account_name = {account_name}") + + self.library_name = library_name + self.account_name = account_name + self.library_main_path = os.path.join(LLMWareConfig.get_library_path(), account_name, library_name) + + # add new file dir for this collection + self.file_copy_path = os.path.join(self.library_main_path, "uploads" + os.sep) + self.image_path = os.path.join(self.library_main_path, "images" + os.sep) + self.dataset_path = os.path.join(self.library_main_path, "datasets" + os.sep) + self.nlp_path = os.path.join(self.library_main_path, "nlp" + os.sep) + self.output_path = os.path.join(self.library_main_path, "output" + os.sep) + self.tmp_path = os.path.join(self.library_main_path, "tmp" + os.sep) + self.embedding_path = os.path.join(self.library_main_path, "embedding" + os.sep) + + os.makedirs(self.library_main_path, exist_ok=True) + os.makedirs(self.file_copy_path,exist_ok=True) + os.makedirs(self.image_path,exist_ok=True) + os.makedirs(self.dataset_path,exist_ok=True) + os.makedirs(self.nlp_path,exist_ok=True) + os.makedirs(self.output_path,exist_ok=True) + os.makedirs(self.tmp_path,exist_ok=True) + os.makedirs(self.embedding_path,exist_ok=True) + + return self + + def get_library_card(self, library_name=None, account_name="llmware"): + + """Retrieves the library card dictionary with key attributes of library. + + Parameters + ---------- + library_name : str, default=None + Name of the library to retrieve. If not provided, uses self.library_name + + account_name : str, default="llmware" + Name of the account associated to the library. If not provided, uses self.account_name + + Returns + ------- + library_card : dict or None + The library card dictionary containing key atrributes of the library. If not found, returns None + """ + + library_card = None + + if library_name: + lib_lookup_name = library_name + acct_lookup_name = account_name + else: + lib_lookup_name = self.library_name + acct_lookup_name = self.account_name + + if lib_lookup_name and acct_lookup_name: + library_card= LibraryCatalog().get_library_card(lib_lookup_name, account_name=acct_lookup_name) + + if not library_card: + logger.warning(f"Library - get_library_card - library card not found - {library_name} - {account_name}") + + return library_card + + def check_if_library_exists(self, library_name, account_name="llmware"): + + """Check if library exists by library string name. + + Parameters + ---------- + library_name : str + Name of library to check. + + account_name : str, default="llmware" + Name of account associated with library. + + Returns + ------- + library_card : dict or None + The library card dict if the library exists. If not found, returns None. + + """ + + # first look in library catalog + library_card = LibraryCatalog().get_library_card(library_name, account_name=account_name) + + # check file path + lib_path = os.path.join(LLMWareConfig.get_library_path(), account_name, library_name) + library_folder = os.path.exists(lib_path) + + # if all checks consistent + if library_card and library_folder: + # library exists and is in good state + return library_card + + if not library_card and not library_folder: + # library does not exist conclusively + return None + + # may be error state - some artifacts exist and others do not + if library_card: + # view the library_card as the definitive record + return library_card + + return library_card + + def update_embedding_status (self, status_message, embedding_model, embedding_db, + embedded_blocks=0, embedding_dims=0,time_stamp="NA",delete_record=False): + + """Invoked at the end of the embedding job to update the library card and embedding record -- generally, + this method does not need to be invoked directly. + + Parameters + ---------- + status_message : str + Status message for the embedding process. If "delete", the record will be marked for deletion. + + embedding_model : str + Name of the embedding model used. + + embedding_db : str + Name of the embedding database used. + + embedded_blocks : int, default=0 + Number of embedded blocks. + + embedding_dims : int, default=0 + Dimensions of the embedding. + + time_stamp : str, default="NA" + Timestamp of the embedding process. + + delete_record : bool, default=False + If True, marks the record for deletion. + + Returns + ------- + bool + True if the embedding status was successfully updated. + """ + + # special handling for updating "embedding" in update_library_card + # -- append/insert this new embedding dict to the end of the embedding list + + if status_message == "delete": + delete_record = True + + update_dict = {"embedding": {"embedding_status": status_message, + "embedding_model": embedding_model, + "embedding_db": embedding_db, + "embedding_dims": embedding_dims, + "embedded_blocks": embedded_blocks, + "time_stamp": time_stamp}} + + updater = LibraryCatalog(self).update_library_card(self.library_name, update_dict, + delete_record=delete_record, account_name=self.account_name) + + return True + + def get_embedding_status (self): + + """Pulls the embedding record for the current library from the library card. + + Returns + ------- + embedding_record : list or None + The embedding record, which is a list of dictionaries containing embedding status, model, and database. + If the library card or embedding record is not found, returns None. + """ + + library_card = LibraryCatalog(self).get_library_card(self.library_name, account_name=self.account_name) + + if not library_card: + + raise LLMWareException(message=f"Library - get_embedding_status - library not found -" + f"library_name = {self.library_name} with account_name = {self.account_name}") + + # embedding record will be a list of {"embedding_status" | "embedding_model" | "embedding_db"} + logger.info(f"Library - get_embedding_status - library_card - {library_card}") + + if "embedding" in library_card: + embedding_record = library_card["embedding"] + else: + logger.warning(f"Library - get_embedding_status - could not identify embedding record in library card - {library_card}") + embedding_record = None + + return embedding_record + + def get_and_increment_doc_id(self): + + """Convenience method in library class - mirrors method in LibraryCatalog - increments, tracks and provides a + unique doc id for the library. + + Returns + ------- + unique_doc_id : int + The new unique document ID for the library. + """ + + unique_doc_id = LibraryCatalog(self).get_and_increment_doc_id(self.library_name) + return unique_doc_id + + def set_incremental_docs_blocks_images(self, added_docs=0, added_blocks=0, added_images=0, added_pages=0, + added_tables=0): + + """Updates the library card with incremental counters after completing a parsing job. + + Parameters + ---------- + added_docs : int, default=0 + Number of documents added. + + added_blocks : int, default=0 + Number of blocks added. + + added_images : int, default=0 + Number of images added. + + added_pages : int, default=0 + Number of pages added. + + added_tables : int, default=0 + Number of tables added. + + Returns + ------- + bool + True if the incremental counters were successfully updated. + """ + + # updates counting parameters at end of parsing + updater = LibraryCatalog(self).set_incremental_docs_blocks_images(added_docs=added_docs, + added_blocks=added_blocks, + added_images=added_images, + added_pages=added_pages, + added_tables=added_tables) + + return True + + def add_file(self, file_path): + + """Ingests, parses, text chunks and indexes a single selected file to a library - + provide the full path to file. + + Parameters + ---------- + file_path : str + The full path to the file to be ingested and indexed. + + Returns + ------- + self : Library + The updated ``Library`` object after adding the file. + """ + + # Ensure the input path exists + os.makedirs(LLMWareConfig.get_input_path(), exist_ok=True) + + file_name = os.path.basename(file_path) + target_path = os.path.join(LLMWareConfig.get_input_path(), file_name) + + shutil.copyfile(file_path,target_path) + return self.add_files() + + def add_files (self, input_folder_path=None, encoding="utf-8",chunk_size=400, + get_images=True,get_tables=True, smart_chunking=1, max_chunk_size=600, + table_grid=True, get_header_text=True, table_strategy=1, strip_header=False, + verbose_level=2, copy_files_to_library=True, set_custom_logging=-1, + use_logging_file=False): + + """Main method to integrate documents into a Library - pass a local filepath folder and all files will be + routed to appropriate parser by file type extension. + + Parameters + ---------- + input_folder_path : str, default=None + The path to the folder containing files to be ingested. If not provided, defaults to None. + + encoding : str, default="utf-8" + The encoding to use for reading files. + + chunk_size : int, default=400 + The size of text chunks to create during parsing. + + get_images : bool, default=True + Whether to extract images from the documents. + + get_tables : bool, default=True + Whether to extract tables from the documents. + + smart_chunking : int, default=1 + The strategy for smart chunking of text. + + max_chunk_size : int, default=600 + The maximum size of text chunks. + + table_grid : bool, default=True + Whether to use a grid for tables. + + get_header_text : bool, default=True + Whether to extract header text from the documents. + + table_strategy : int, default=1 + The strategy to use for table extraction. + + strip_header : bool, default=False + Whether to strip headers from the documents. + + verbose_level : int, default=2 + The level of verbosity for logging. + + copy_files_to_library : bool, default=True + Whether to copy the files to the library. + + set_custom_logging : int, default=-1, will apply a custom logging level between 0-50 for the + parsing job. + + use_logging_file : bool, default=False + Whether parse should log to stdout (default) or to file (set to True) + + Returns + ------- + output_results : dict or None + A dictionary containing the results of the document integration process, including counts of added documents, + blocks, images, pages, tables, and rejected files. If the library card could not be identified, returns None. + """ + + if not input_folder_path: + input_folder_path = LLMWareConfig.get_input_path() + + # get overall counters at start of process + lib_counters_before = self.get_library_card() + + parsing_results = Parser(library=self, + encoding=encoding, + chunk_size=chunk_size, + max_chunk_size=max_chunk_size, + smart_chunking=smart_chunking, + get_tables=get_tables, + get_images=get_images, + get_header_text=get_header_text, + table_strategy=table_strategy, + strip_header=strip_header, + table_grid=table_grid, + verbose_level=verbose_level, + copy_files_to_library=copy_files_to_library, + set_custom_logging=set_custom_logging, + use_logging_file=use_logging_file).ingest(input_folder_path,dupe_check=True) + + logger.debug(f"Library - add_files - parsing results - {parsing_results}") + + # post-processing: get the updated lib_counters + lib_counters_after = self.get_library_card() + + # parsing_results = {"processed_files" | "rejected_files" | "duplicate_files"} + output_results = None + + if lib_counters_after and lib_counters_before: + output_results = {"docs_added": lib_counters_after["documents"] - lib_counters_before["documents"], + "blocks_added": lib_counters_after["blocks"] - lib_counters_before["blocks"], + "images_added": lib_counters_after["images"] - lib_counters_before["images"], + "pages_added": lib_counters_after["pages"] - lib_counters_before["pages"], + "tables_added": lib_counters_after["tables"] - lib_counters_before["tables"], + "rejected_files": parsing_results["rejected_files"]} + else: + logger.error("Library - add_files - unexpected - could not identify the library_card correctly") + + logger.info(f"Library - add_files - output_results - {output_results}") + + # update collection text index in collection after adding documents + # LibraryCollection(self).create_index() + CollectionWriter(self.library_name,account_name=self.account_name).build_text_index() + + return output_results + + def export_library_to_txt_file(self, output_fp=None, output_fn=None, include_text=True, include_tables=True, + include_images=False): + + """Exports library collection of indexed text chunks to a txt file. + + Parameters + ---------- + output_fp : str, default=None + The file path where the output file will be saved. If not provided, defaults to None. + + output_fn : str, default=None + The name of the output file. If not provided, defaults to None. + + include_text : bool, default=True + Whether to include text content in the export. + + include_tables : bool, default=True + Whether to include tables in the export. + + include_images : bool, default=False + Whether to include images in the export. + + Returns + ------- + file_location : str + The location of the exported txt file. + """ + + if not output_fp: + output_fp = self.output_path + + if not output_fn: + output_fn = self.library_name + "_" + str(Utilities().get_current_time_now()) + + filter_list = [] + if include_text: filter_list.append("text") + if include_tables: filter_list.append("table") + if include_images: filter_list.append("image") + + if not filter_list: + # go with default - text only + filter_list = ["text"] + + results = CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key_value_range("content_type",filter_list) + + file_location = os.path.join(output_fp, output_fn + ".txt") + output_file = open(file_location, "w", encoding='utf-8') + text_field = "text_search" + for elements in results: + new_entry = elements[text_field].strip() + "\n" + output_file.write(new_entry) + + output_file.close() + + return file_location + + def export_library_to_jsonl_file(self, output_fp, output_fn, include_text=True, include_tables=True, + include_images=False, dict_keys=None): + + """Exports collection of text chunks to a jsonl file. + + Parameters + ---------- + output_fp : str + The file path where the output file will be saved. + + output_fn : str + The name of the output file. + + include_text : bool, default=True + Whether to include text content in the export. + + include_tables : bool, default=True + Whether to include tables in the export. + + include_images : bool, default=False + Whether to include images in the export. + + dict_keys : list of str, default=None + The keys to include in the JSONL entries. If not provided, defaults to None. + + Returns + ------- + file_location : str + The location of the exported JSONL file. + """ + + if not output_fp: + output_fp = self.output_path + + if not output_fn: + output_fn = self.library_name + "_" + str(Utilities().get_current_time_now()) + + # expects dict_keys to be a list of dictionary keys + if not dict_keys: + dict_keys = self.default_keys + + filter_list = [] + if include_text: filter_list.append("text") + if include_tables: filter_list.append("table") + if include_images: filter_list.append("image") + + if not filter_list: + # go with default - text only + filter_list = ["text"] + + results = CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key_value_range("content_type", filter_list) + + file_location = os.path.join(output_fp, output_fn + ".jsonl") + output_file = open(file_location, "w", encoding='utf-8') + + for elements in results: + + # package up each jsonl entry as dict with selected keys to extract + new_dict_entry = {} + for keys in dict_keys: + if keys in elements: + new_dict_entry.update({keys:elements[keys]}) + + if new_dict_entry: + jsonl_row = json.dumps(new_dict_entry) + output_file.write(jsonl_row) + output_file.write("\n") + + output_file.close() + + return file_location + + def pull_files_from_cloud_bucket (self, aws_access_key=None, aws_secret_key=None, bucket_name=None): + + """Pull files from private S3 bucket into local cache for further processing. + + Parameters + ---------- + aws_access_key : str, default=None + The AWS access key for connecting to the S3 bucket. + + aws_secret_key : str, default=None + The AWS secret key for connecting to the S3 bucket. + + bucket_name : str, default=None + The name of the S3 bucket from which to pull files. + + Returns + ------- + files_copied : list + A list of file paths that were copied from the S3 bucket to the local cache. + """ + + files_copied = CloudBucketManager().connect_to_user_s3_bucket (aws_access_key, aws_secret_key, + bucket_name, LLMWareConfig.get_input_path()) + + return files_copied + + def install_new_embedding (self, embedding_model_name=None, vector_db=None, + from_hf= False, from_sentence_transformer=False, model=None, tokenizer=None, model_api_key=None, + vector_db_api_key=None, batch_size=500, max_len=None, use_gpu=True): + + """Main method for installing a new embedding on a library. + + Parameters + ---------- + embedding_model_name : str, default=None + The name of the embedding model to use. + + vector_db : str, default=None + The name of the vector database to use. + + from_hf : bool, default=False + Whether the model is from Hugging Face. + + from_sentence_transformer : bool, default=False + Whether the model is a Sentence Transformer. + + model : object, default=None + The pre-loaded model to use. + + tokenizer : object, default=None + The tokenizer associated with the pre-loaded model. + + model_api_key : str, default=None + The API key for accessing the model. + + vector_db_api_key : str, default=None + The API key for accessing the vector database. + + batch_size : int, default=500 + The batch size to use for embedding. + + max_len : int, default=None + The maximum length for embedding. + + use_gpu : bool, default=True + Whether to use GPU for embedding. + + Returns + ------- + embeddings : dict or None + The created embeddings dict, or None if no embeddings could be created. + """ + + embeddings = None + my_model = None + + # step 1 - load selected model from ModelCatalog - will pass 'loaded' model to the EmbeddingHandler + + # check if instantiated model and tokenizer -> load as HuggingFace model + if model: + if from_hf: + logger.info("Library - install_new_embedding - loading hf model") + my_model = ModelCatalog().load_hf_embedding_model(model, tokenizer) + batch_size = 50 + + if from_sentence_transformer: + logger.info("library - install_new_embedding - loading sentence transformer model") + if not embedding_model_name: + raise LLMWareException(message=f"Library - install_new_embedding - to use " + f"sentence_transformer model requires providing the model name.") + + my_model = ModelCatalog().load_sentence_transformer_model(model,embedding_model_name) + else: + # if no model explicitly passed, then look up in the model catalog + if embedding_model_name: + my_model = ModelCatalog().load_model(selected_model=embedding_model_name, api_key=model_api_key) + + if not my_model: + logger.error("Library - install_new_embedding - can not identify a selected model") + return -1 + + # new - insert - handle no vector_db passed + if not vector_db: + vector_db = LLMWareConfig().get_config("vector_db") + # end - new insert + + if vector_db not in LLMWareConfig().get_supported_vector_db(): + raise LLMWareException(message=f"Library - install_new_embedding - selected " + f"vector db is not supported - {vector_db}") + + if my_model and max_len: + my_model.max_len = max_len + + # step 2 - pass loaded embedding model to EmbeddingHandler, which will route to the appropriate resource + embeddings = EmbeddingHandler(self).create_new_embedding(vector_db, my_model, batch_size=batch_size) + + if not embeddings: + logger.warning("Library - install_new_embedding - no embeddings created") + + return embeddings + + def delete_library(self, library_name=None, confirm_delete=False, account_name="llmware"): + + """ Deletes all artifacts of a library + + Parameters + ---------- + library_name : str, default=None + The name of the library to delete. If not provided, defaults to None. + + confirm_delete : bool, default=False + Confirmation flag to proceed with deletion. Must be set to True to delete the library. + + Returns + ------- + success_code : int + Returns 1 if the deletion was successful, or -1 if an error occurred. + + """ + + if library_name: + self.library_name = library_name + + # loads the library specific path information if required + self.load_library(library_name,account_name=account_name) + + success_code = 1 + + try: + if confirm_delete: + + # 1st - remove the blocks - drop the collection in database + CollectionWriter(self.library_name, account_name=self.account_name).destroy_collection(confirm_destroy=True) + + # 2nd - Eliminate the local file structure + file_path = self.library_main_path + shutil.rmtree(file_path) + + # 3rd - remove record in LibraryCatalog + LibraryCatalog(self).delete_library_card(self.library_name) + + logger.info("Library - delete_library - deleted all library file artifacts + folders") + + except: + logger.exception("Library - delete_library - error destroying library") + success_code = -1 + + return success_code + + def update_block (self, doc_id, block_id, key, new_value): + + """Convenience method to update the record of a specific block - identified by doc_ID and block_ID + in text collection database. + + Parameters + ---------- + doc_id : int + The ID of the document containing the block to update. + + block_id : int + The ID of the block to update. + + key : str + The key in the block record to update. + + new_value : str + The new value to set for the specified key. + + Returns + ------- + completed : bool + True if the block was successfully updated, False otherwise. + """ + + completed = (CollectionWriter(self.library_name, account_name=self.account_name). + update_block(doc_id, block_id,key,new_value,self.default_keys)) + + return completed + + def add_website (self, url, get_links=True, max_links=5): + + """Main method to ingest a website into a library. + + Parameters + ---------- + url : str + The URL of the website to ingest. + + get_links : bool, default=True + Whether to follow and ingest links found on the website. + + max_links : int, default=5 + The maximum number of links to follow and ingest. + + Returns + ------- + self : Library + The updated ``Library`` object after ingesting the website. + """ + + Parser(library=self).parse_website(url,get_links=get_links,max_links=max_links) + CollectionWriter(self.library_name, account_name=self.account_name).build_text_index() + + return self + + def add_wiki(self, topic_list,target_results=10): + + """Main method to add a wikipedia article to a library - enter a list of topics. + + Parameters + ---------- + topic_list : list of str + A list of topics to search for on Wikipedia. + + target_results : int, default=10 + The target number of results to retrieve for each topic. + + Returns + ------- + self : Library + The updated ``Library`` object after adding the Wikipedia articles. + """ + + Parser(library=self).parse_wiki(topic_list,target_results=target_results) + CollectionWriter(self.library_name, account_name=self.account_name).build_text_index() + + return self + + def add_dialogs(self, input_folder=None): + + """Main method to add an AWS dialog transcript into a library. + + Parameters + ---------- + input_folder : str, default=None + The path to the folder containing the dialog transcripts. If not provided, defaults to None. + + Returns + ------- + self : Library + The updated ``Library`` object after adding the dialog transcripts. + """ + + if not input_folder: + input_folder = LLMWareConfig.get_input_path() + + output = Parser(library=self).parse_dialog(input_folder) + + return self + + def add_image(self, input_folder=None): + + """Main method to add image and scanned OCR content into a library. + + Parameters + ---------- + input_folder : str, default=None + The path to the folder containing the images. If not provided, defaults to None + + Returns + ------- + self : Library + The updated ``Library`` object after adding the image and OCR content. + """ + + if not input_folder: + input_folder = LLMWareConfig.get_input_path() + + output = Parser(library=self).parse_image(input_folder) + + return self + + def add_pdf_by_ocr(self, input_folder=None): + + """Alternative method to ingest PDFs that are scanned, or can not be otherwise parsed. + + Parameters + ---------- + input_folder : str, default=None + The path to the folder containing the PDFs. If not provided, defaults to None + + Returns + ------- + self : Library + The updated ``Library`` object after adding the PDFs through OCR. + """ + + if not input_folder: + input_folder = LLMWareConfig.get_input_path() + + output = Parser(library=self).parse_pdf_by_ocr_images(input_folder) + + return self + + def add_pdf(self, input_folder=None): + + """Convenience method to directly add PDFs only - note, in most cases, 'add_files' is the better option. + + Parameters + ---------- + input_folder : str, default=None + The path to the folder containing the PDFs. If not provided, defaults to None + + Returns + ------- + self : Library + The updated ``Library`` object after adding the PDFs. + """ + + if not input_folder: + input_folder = LLMWareConfig.get_input_path() + + output = Parser(library=self).parse_pdf(input_folder) + + return self + + def add_office(self, input_folder=None): + + """Convenience method to directly add PDFs only - note, in most cases, 'add_files' is the better option. + + Parameters + ---------- + input_folder : str, default=None + The path to the folder containing the Office documents. If not provided, defaults to None. + + Returns + ------- + self : Library + The updated ``Library`` object after adding the Office documents. + """ + + if not input_folder: + input_folder = LLMWareConfig.get_input_path() + + output = Parser(library=self).parse_office(input_folder) + + return self + + def get_all_library_cards(self, account_name='llmware'): + + """Get all library cards for all libraries on account. + + Parameters + ---------- + account_name : str, default='llmware' + The name of the account for which to retrieve all library cards. + + Returns + ------- + library_cards : list of dict + A list of all library card dictionaries for the specified account. + """ + + library_cards = LibraryCatalog(account_name=account_name).all_library_cards() + return library_cards + + def delete_installed_embedding(self, embedding_model_name, vector_db, vector_db_api_key=None): + + """Deletes an installed embedding on specific combination of vector_db + embedding_model_name. + + Parameters + ---------- + embedding_model_name : str + The name of the embedding model to delete. + + vector_db : str + The name of the vector database from which to delete the embedding. + + vector_db_api_key : str, default=None + The API key for accessing the vector database. If not provided, defaults to None + + Returns + ------- + int + Returns 1 if the embedding was successfully deleted. + """ + + # insert safety check - confirm that this is valid combination with installed embedding + lib_card = LibraryCatalog(self).get_library_card(self.library_name) + embedding_list = lib_card["embedding"] + found_match = False + embedding_dims = 0 + + for entries in embedding_list: + if entries["embedding_model"] == embedding_model_name and entries["embedding_db"] == vector_db: + found_match = True + embedding_dims = entries["embedding_dims"] + + logger.info(f"Library - delete_installed_embedding - found matching" + f"embedding record - {entries}") + break + + if found_match: + EmbeddingHandler(self).delete_index(vector_db,embedding_model_name, embedding_dims) + else: + # update exception + raise LLMWareException(message=f"Library - delete_installed_embedding - library not found -" + f"library_name = {self.library_name} with account_name = {self.account_name}") + + return 1 + + def run_ocr_on_images(self, add_to_library=False,chunk_size=400,min_size=10, realtime_progress=True): + + """Convenience method in Library class to pass Library to Parser to run OCR on all of the images + found in the Library, and OCR-extracted text from the images directly into the Library as additional + blocks. + + Parameters + ---------- + add_to_library : bool, default=False + Whether to add the OCR-extracted text directly into the Library as additional blocks. + + chunk_size : int, default=400 + The size of text chunks to create during OCR processing. + + min_size : int, default=10 + The minimum size of text chunks to consider during OCR processing. + + realtime_progress : bool, default=True + Whether to display real-time progress during OCR processing. + + Returns + ------- + output : int + Returns 1 if running the OCR on the images was successful. + """ + + output = Parser(library=self).ocr_images_in_library(add_to_library=add_to_library, + chunk_size=chunk_size,min_size=min_size, + realtime_progress=realtime_progress) + + return output + + def expand_text_result_before(self, block, window_size=400): + + """ Expands text result before. Duplicate of Query method - added to Library class for direct access and + convenience in certain use cases.""" + + block_id = block["block_ID"] -1 + doc_id = block["doc_ID"] + + before_text = "" + pre_blocks = [] + + while len(before_text) < window_size and block_id >= 0: + + before_block = self.block_lookup(block_id, doc_id) + + if before_block: + before_text += before_block["text"] + pre_blocks.append(before_block) + + output = {"expanded_text": before_text, "results": pre_blocks} + + return output + + def expand_text_result_after(self, block, window_size=400): + + """ Expands text result after. Duplicate of Query method - added to Library class for direct access and + convenience in certain use cases. """ + + block_id = block["block_ID"] + 1 + doc_id = block["doc_ID"] + + after_text = "" + post_blocks = [] + + while len(after_text) < window_size: + after_block = self.block_lookup(block_id, doc_id) + if not after_block: + break # Break if no block is found + + after_text += after_block["text"] + post_blocks.append(after_block) + block_id += 1 # Increment block_id for next iteration + + output = {"expanded_text": after_text, "results": post_blocks} + return output + + def block_lookup(self, block_id, doc_id): + + """ Look up by a specific pair of doc_id and block_id in a library. Duplicate of Query method - + added to Library class for direct access and convenience in certain use cases.""" + + result = None + + kv_dict = {"doc_ID": doc_id, "block_ID": block_id} + + output = CollectionRetrieval(self.library_name, account_name=self.account_name).filter_by_key_dict(kv_dict) + + if len(output) == 0: + logger.info(f"Library - block_lookup - block not found: {block_id}") + result = None + + return result + + if len(output) > 1: + result = output[0] + + if len(output) == 1: + result = output[0] + + # if arrived this point, then positive result has been identified + result.update({"matches": []}) + result.update({"page_num": result["master_index"]}) + + return result + + +class LibraryCatalog: + + """Implements the management of tracking details for libraries via the library card, which is stored + in the `library` table of the text collection database. It is used by the ``Library`` class. + + ``LibraryCatalog`` is responsible for managing tracking details. This includes creating, + reading, updating, and deleting library cards. The library card is stored in the table library + of the chosen text collection database. In most cases, ``LibraryCatalog`` does not need to be directly + invoked, instead it is used indirectly through the methods of ``Library``. + + Parameters + ---------- + library : Library, default=None + The library with which the ``LibraryCatalog`` interacts. + + library_path : str or pathlib.Path object, default=None + The path to the llmware directory. If set, then the default from ``LLMWareconfig`` is used. + + account_name : str, default='llmware' + Name of the account. + + Returns + ------- + library_catalog : LibraryCatalog + A new ``LibraryCatalog`` object. + """ + + def __init__(self, library=None, library_path=None, account_name="llmware"): + + self.library = library + if library: + self.library_name = library.library_name + self.account_name = library.account_name + else: + self.library_name = None + self.account_name = account_name + + self.schema = LLMWareTableSchema().get_library_card_schema() + + # if table does not exist, then create + if CollectionWriter("library",account_name=self.account_name).check_if_table_build_required(): + CollectionWriter("library", account_name=self.account_name).create_table("library", self.schema) + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + + if not library_path: + self.library_path = LLMWareConfig.get_llmware_path() + else: + self.library_path = library_path + + def get_library_card (self, library_name, account_name="llmware"): + + """ Gets the selected library card for the selected library_name """ + + # note: will return either library_card {} or None + + db_record = CollectionRetrieval("library", account_name=account_name).lookup("library_name", library_name) + + if isinstance(db_record, list): + if len(db_record) > 0: + db_record = db_record[0] + + library_card = db_record + + return library_card + + def all_library_cards(self): + + """ Get all library cards """ + + all_library_cards_cursor = CollectionRetrieval("library", + account_name=self.account_name).get_whole_collection() + + all_library_cards = all_library_cards_cursor.pull_all() + + return all_library_cards + + def create_new_library_card(self, new_library_card): + + """ Creates new library card entry """ + + new_lib_name = new_library_card["library_name"] + + logger.debug(f"LibraryCatalog - create_new_library_card - {new_lib_name} - {new_library_card}") + + CollectionWriter("library", account_name=self.account_name).write_new_record(new_library_card) + + # test to pull card here + # lib_card = self.get_library_card(new_lib_name) + # end - test get card + + return 0 + + def update_library_card(self, library_name, update_dict, account_name="llmware", delete_record=False): + + """ Updates library card entry """ + + lib_card = self.get_library_card(library_name, account_name=account_name) + + updater = CollectionWriter("library", + account_name=self.account_name).update_library_card(library_name, + update_dict, + lib_card, + delete_record=delete_record) + + return 1 + + def delete_library_card(self, library_name=None, account_name="llmware"): + + """ Deletes library card """ + + if not library_name: + library_name = self.library_name + + if account_name != "llmware": + self.account_name = account_name + + f = {"library_name": library_name} + + # self.library_card_collection.delete_one(f) + CollectionWriter("library", account_name=self.account_name).delete_record_by_key("library_name", library_name) + + return 1 + + def get_and_increment_doc_id (self, library_name, account_name="llmware"): + + """ Gets and increments unique doc id counter for library """ + + if account_name != "llmware": + self.account_name = account_name + + cw = CollectionWriter("library", account_name=self.account_name) + unique_doc_id = cw.get_and_increment_doc_id(library_name) + + return unique_doc_id + + def set_incremental_docs_blocks_images(self, added_docs=0, added_blocks=0, added_images=0, added_pages=0, + added_tables=0): + + """ Updates library card with incremental counters after parsing """ + + # updates counting parameters at end of parsing + cw = CollectionWriter("library", account_name=self.account_name) + + cw.set_incremental_docs_blocks_images(self.library_name,added_docs=added_docs,added_blocks=added_blocks, + added_images=added_images, added_pages=added_pages, + added_tables=added_tables) + return 0 + diff --git a/llmware/model_configs.py b/llmware/model_configs.py new file mode 100644 index 0000000..c3cbeaf --- /dev/null +++ b/llmware/model_configs.py @@ -0,0 +1,4617 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + + +"""Global Default Configs for Models, Finetune Wrappers and Prompt Instructions Catalog. + +These configs generally do not need to be accessed directly, but should be viewed, accessed and modified through +ModelCatalog and PromptCatalog classes. + +For customization, there is also the option in ModelCatalog to load a custom model catalog from json file, which +would over-write this list. +""" + +global_model_repo_catalog_list = [ + + {"model_name": "bling-tiny-llama-onnx", "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", "display_name": "llmware/bling-tiny-llama-onnx", + "model_location": "llmware_repo","context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/bling-tiny-llama-onnx", "custom_model_files": [], "custom_model_repo": "", + "snapshot": True, "tokenizer_local": "tokenizer_tl.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/bling-tiny-llama-onnx"}, + + {"model_name": "bling-tiny-llama-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "bling-tiny-llama-ov", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_tl.json", + "hf_repo": "llmware/bling-tiny-llama-ov", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml"], + "link": "https://huggingface.co/llmware/bling-tiny-llama-ov"}, + + {"model_name": "bling-phi-3-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "bling-phi-3-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/bling-phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models","method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml"], + "link": "https://huggingface.co/llmware/bling-phi-3-ov"}, + + {"model_name": "bling-phi-3-onnx", "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", "display_name": "bling-phi-3-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/bling-phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/bling-phi-3-onnx"}, + + {"model_name": "phi-3-onnx", "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/phi-3-onnx"}, + + {"model_name": "phi-3-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml"], + "link": "https://huggingface.co/llmware/phi-3-ov"}, + + # new text-to-image model - more coming soon + {"model_name": "lcm-dreamshaper-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "lcm-dreamshaper-ov", + "model_location": "llmware_repo", "pipeline": "text2image", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/lcm-dreamshaper-ov", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "link": "https://huggingface.co/llmware/lcm-dreamshaper-ov"}, + + {"model_name": "qwen2.5-1.5b-instruct-ov", "display_name": "qwen2.5-1.5b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-1.5b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-1.5b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-0.5b-instruct-ov", "display_name": "qwen2.5-0.5b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2-0.5b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-0.5b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-3b-instruct-ov", "display_name": "qwen2.5-3b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2-3b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-3b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-7b-instruct-v0.3-ov", "display_name": "mistral-7b-instruct-v0.3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-7b-instruct-v0.3-ov", + "link": "https://huggingface.co/llmware/mistral-7b-instruct-v0.3-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-llama2-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-llama2-ov", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-llama2-ov", + "tokenizer_local": "tokenizer_ll2.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models","method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-llama2-ov"}, + + {"model_name": "dragon-mistral-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-mistral-ov", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-mistral-ov", + "tokenizer_local": "tokenizer_mistral.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-mistral-ov"}, + + {"model_name": "dragon-yi-9b-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-yi-9b-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-yi-9b-ov", + "tokenizer_local": "tokenizer_yi.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-yi-9b-ov"}, + + {"model_name": "slim-extract-tiny-ov", "display_name": "slim-extract-tiny-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-extract-tiny-ov", + "link": "https://huggingface.co/llmware/slim-extract-tiny-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "function": ["extract"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-extract-phi-3-ov", "display_name": "slim-extract-phi-3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-extract-phi-3-ov", + "link": "https://huggingface.co/llmware/slim-extract-phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["extract"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-sentiment-ov", "display_name": "slim-sentiment-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-sentiment-ov", + "link": "https://huggingface.co/llmware/slim-sentiment-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["sentiment"], + "fc_output_values": ["positive", "neutral", "negative"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["classify"], + "snapshot": True, + "fetch": {"module": "llmware.models","method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + # embedding models + + {"model_name": "all-MiniLM-L6-v2", "display_name": "mini-lm-sbert", "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 384, "context_window": 512, + "link": "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "sentence-transformers/all-MiniLM-L6-v2"}, + + {"model_name": 'all-mpnet-base-v2', "display_name": "mpnet-base", "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window": 514, + "link": "https://huggingface.co/sentence-transformers/all-mpnet-base-v2", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "sentence-transformers/all-mpnet-base-v2"}, + + {"model_name": 'industry-bert-insurance', "display_name": "industry-bert-insurance", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window":512, + "link": "https://huggingface.co/llmware/industry-bert-insurance-v0.1", "custom_model_files":[], + "custom_model_repo": "", + "hf_repo": "llmware/industry-bert-insurance-v0.1"}, + + {"model_name": 'industry-bert-contracts', "display_name": "industry-bert-contracts", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window":512, + "link": "https://huggingface.co/llmware/industry-bert-contracts-v0.1", "custom_model_files":[], + "custom_model_repo": "", + "hf_repo": "llmware/industry-bert-contracts-v0.1"}, + + {"model_name": 'industry-bert-asset-management', "display_name": "industry-bert-asset-management", + "model_family": "HFEmbeddingModel", "model_category": "embedding", "model_location": "hf_repo", + "embedding_dims": 768, "context_window":512, + "link": "https://huggingface.co/llmware/industry-bert-asset-management-v0.1", "custom_model_files":[], + "custom_model_repo": "", + "hf_repo": "llmware/industry-bert-asset-management-v0.1"}, + + {"model_name": 'industry-bert-sec', "display_name": "industry-bert-sec", "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window":512, + "link": "https://huggingface.co/llmware/industry-bert-sec-v0.1", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/industry-bert-sec-v0.1"}, + + {"model_name": 'industry-bert-loans', "display_name": "industry-bert-loans", + "model_family": "HFEmbeddingModel", "model_category": "embedding", "model_location": "hf_repo", + "embedding_dims": 768, "context_window": 512, + "link": "https://huggingface.co/llmware/industry-bert-loans", + "custom_model_files": [], "custom_model_repo": "", "hf_repo": "llmware/industry-bert-loans"}, + + {"model_name": 'nomic-ai/nomic-embed-text-v1', "display_name": "nomic-text-v1", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window": 8192, + "link": "https://huggingface.co/nomic-ai/nomic-embed-text-v1", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "nomic-ai/nomic-embed-text-v1"}, + + {"model_name": 'jinaai/jina-embeddings-v2-base-en', "display_name": "jina-base-en-v2", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window": 8192, + "link": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "jinaai/jina-embeddings-v2-base-en"}, + + {"model_name": 'jinaai/jina-embeddings-v2-small-en', "display_name": "jina-small-en-v2", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 512, "context_window": 8192, + "link": "https://huggingface.co/jinaai/jina-embeddings-v2-small-en", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "jinaai/jina-embeddings-v2-small-en"}, + + {"model_name": 'jinaai/jina-reranker-v1-turbo-en', "display_name": "jina-reranker-turbo", + "model_family": "HFReRankerModel", + "model_category": "reranker", "model_location": "hf_repo", "embedding_dims": 384, "context_window": 8192, + "link": "https://huggingface.co/jinaai/jina-reranker-v1-turbo-en", "custom_model_files": [], + "custom_model_repo": "", + "hf_repo": "jinaai/jina-reranker-v1-turbo-en"}, + + {"model_name": 'jinaai/jina-reranker-v1-tiny-en', "display_name": "jina-reranker-tiny", + "model_family": "HFReRankerModel", + "model_category": "reranker", "model_location": "hf_repo", "embedding_dims": 384, "context_window": 8192, + "link": "https://huggingface.co/jinaai/jina-reranker-v1-tiny-en", "custom_model_files": [], + "custom_model_repo": "", + "hf_repo": "jinaai/jina-reranker-v1-tiny-en"}, + + {"model_name": 'BAAI/bge-small-en-v1.5', "display_name": "bge-small-en-v1.5", "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 384, "context_window": 512, + "link": "https://huggingface.co/BAAI/bge-small-en-v1.5", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "BAAI/bge-small-en-v1.5"}, + + {"model_name": 'BAAI/bge-large-en-v1.5', "display_name": "bge-large-en-v1.5", "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 1024, "context_window": 512, + "link": "https://huggingface.co/BAAI/bge-large-en-v1.5", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "BAAI/bge-large-en-v1.5"}, + + {"model_name": 'BAAI/bge-base-en-v1.5', "display_name": "bge-base-en-v1.5", "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window": 512, + "link": "https://huggingface.co/BAAI/bge-base-en-v1.5", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "BAAI/bge-base-en-v1.5"}, + + {"model_name": "thenlper/gte-small", "display_name": "gte-small", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 384, "context_window": 512, + "link": "https://huggingface.co/thenlper/gte-small", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "thenlper/gte-small"}, + + {"model_name": "thenlper/gte-base", "display_name": "gte-base", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 768, "context_window": 512, + "link": "https://huggingface.co/thenlper/gte-base", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "thenlper/gte-base"}, + + {"model_name": "thenlper/gte-large", "display_name": "gte-large", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 1024, "context_window": 512, + "link": "https://huggingface.co/thenlper/gte-large", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "thenlper/gte-large"}, + + {"model_name": 'llmrails/ember-v1', "display_name": "ember-v1", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 1024, "context_window": 512, + "link": "https://huggingface.co/llmrails/ember-v1", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmrails/ember-v1"}, + + {"model_name": "WhereIsAI/UAE-Large-V1", "display_name": "uae-large-v1", + "model_family": "HFEmbeddingModel", + "model_category": "embedding", "model_location": "hf_repo", "embedding_dims": 1024, "context_window": 512, + "link": "https://huggingface.co/WhereIsAI/UAE-Large-V1", "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "WhereIsAI/UAE-Large-V1"}, + + {"model_name": 'text-embedding-ada-002', "display_name": "OpenAI-Embedding", "model_family": "OpenAIEmbeddingModel", + "model_category": "embedding", "model_location": "api", "context_window": 8191, "embedding_dims": 1536}, + + {"model_name": 'text-embedding-3-small', "display_name": "OpenAI-Embedding", "model_family": "OpenAIEmbeddingModel", + "model_category": "embedding", "model_location": "api", "context_window": 8191, "embedding_dims": 1536}, + + {"model_name": 'text-embedding-3-large', "display_name": "OpenAI-Embedding", "model_family": "OpenAIEmbeddingModel", + "model_category": "embedding", "model_location": "api", "context_window": 8191, "embedding_dims": 3072}, + + {"model_name": "gemini-3-pro-preview", "display_name": "Gemini-3-Pro", + "model_family": "GoogleGeminiModel", + "model_category": "generative-api", "model_location": "api", "context_window": 1000000, "parameters": 100.0}, + + {"model_name": "gemini-3-flash-preview", "display_name": "Gemini-3-Flash", + "model_family": "GoogleGeminiModel", + "model_category": "generative-api", "model_location": "api", "context_window": 1000000, "parameters": 100.0}, + + {"model_name": "gemini-2.5-pro", "display_name": "Gemini-2.5-Pro", + "model_family": "GoogleGeminiModel", + "model_category": "generative-api", "model_location": "api", "context_window": 1000000, "parameters": 100.0}, + + {"model_name": "gemini-2.5-flash", "display_name": "Gemini-2.5-Flash", + "model_family": "GoogleGeminiModel", + "model_category": "generative-api", "model_location": "api", "context_window": 1000000, "parameters": 100.0}, + + {"model_name": "gemini-2.5-flash-lite", "display_name": "Gemini-2.5-Flash-Lite", + "model_family": "GoogleGeminiModel", + "model_category": "generative-api", "model_location": "api", "context_window": 1000000, "parameters": 100.0}, + + {"model_name": "gpt-5.2-pro", "display_name": "GPT-5.2-Pro", "model_family": "OpenAIGenModel", + "model_category": "generative-api", "model_location": "api", "context_window": 400000, "parameters": 100.0}, + + {"model_name": "gpt-5.2", "display_name": "GPT-5.2", "model_family": "OpenAIGenModel", + "model_category": "generative-api", "model_location": "api", "context_window": 400000, "parameters": 100.0}, + + {"model_name": "gpt-5-mini", "display_name": "GPT-5-Mini", "model_family": "OpenAIGenModel", + "model_category": "generative-api", "model_location": "api", "context_window": 400000, "parameters": 100.0}, + + {"model_name": "gpt-5-nano", "display_name": "GPT-5-Nano", "model_family": "OpenAIGenModel", + "model_category": "generative-api", "model_location": "api", "context_window": 400000, "parameters": 100.0}, + + {"model_name": "gpt-4.1", "display_name": "GPT-4.1", "model_family": "OpenAIGenModel", + "model_category": "generative-api", "model_location": "api", "context_window": 32768, "parameters": 100.0}, + + {"model_name": 'claude-opus-4-5', "display_name": "Anthropic-Claude-4.5-Opus", "model_family": "ClaudeModel", + "model_category": "generative-api", "model_location": "api", "context_window": 200000}, + + {"model_name": 'claude-haiku-4-5', "display_name": "Anthropic-Claude-4.5-Haiku", "model_family": "ClaudeModel", + "model_category": "generative-api", "model_location": "api", "context_window": 200000}, + + {"model_name": 'claude-sonnet-4-5', "display_name": "Anthropic-Claude-4.5-Sonnet", "model_family": "ClaudeModel", + "model_category": "generative-api", "model_location": "api", "context_window": 200000}, + + {"model_name": 'claude-sonnet-4-20250514', "display_name": "Anthropic-Claude-4-Sonnet", "model_family": "ClaudeModel", + "model_category": "generative-api", "model_location": "api", "context_window": 8192}, + + {"model_name": 'claude-opus-4-20250514', "display_name": "Anthropic-Claude-4-Opus", + "model_family": "ClaudeModel", + "model_category": "generative-api", "model_location": "api", "context_window": 8192}, + + # deprecated - will be removing soon + {"model_name": "gpt-4o", "display_name": "GPT-4o", "model_family": "OpenAIGenModel", + "model_category": "generative-api", "model_location": "api", "context_window": 128000}, + + # deprecated - will be removing soon + {"model_name": "o4-mini", "display_name": "gpt-o4-mini", + "model_family": "OpenAIGenModel", "model_category": "generative-api", "model_location": "api", "context_window": 200000}, + + # add api-based llmware custom model + {"model_name": "llmware-inference-server", "display_name": "LLMWare-GPT", "model_family": "LLMWareModel", + "model_category": "generative-api", "model_location": "api", "context_window": 2048}, + + # core llmware bling open source models available in catalog directly + {"model_name": "llmware/bling-1.4b-0.1", "display_name": "bling-1.4b", "model_family": "HFGenerativeModel", + "model_category": "generative_local", "model_location": "hf_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", "temperature": 0.3, "trailing_space":"", + "link": "https://huggingface.co/llmware/bling-1.4b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-1.4b-0.1"}, + + {"model_name": "llmware/bling-1b-0.1", "display_name": "bling-1b", "model_family": "HFGenerativeModel", + "model_category": "generative_local", "model_location": "hf_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", "temperature": 0.3, "trailing_space": "", + "link": "https://huggingface.co/llmware/bling-1b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-1b-0.1"}, + + {"model_name": "llmware/bling-falcon-1b-0.1", "display_name": "bling-falcon-1.3b", "model_family": "HFGenerativeModel", + "model_category": "generative_local", "model_location": "hf_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", "temperature": 0.3, "trailing_space": "", + "link": "https://huggingface.co/llmware/bling-falcon-1b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-falcon-1b-0.1" + }, + + {"model_name": "llmware/bling-sheared-llama-1.3b-0.1", "display_name": "bling-sheared-llama-1.3b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-sheared-llama-1.3b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-sheared-llama-1.3b-0.1" + }, + + {"model_name": "llmware/bling-red-pajamas-3b-0.1", "display_name": "bling-red-pajamas-3b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-red-pajamas-3b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-red-pajamas-3b-0.1"}, + + {"model_name": "llmware/bling-sheared-llama-2.7b-0.1", "display_name": "bling-sheared-llama-2.7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-sheared-llama-2.7b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-sheared-llama-2.7b-0.1"}, + + {"model_name": "llmware/bling-stable-lm-3b-4e1t-v0", "display_name": "bling-stablelm-3b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-stable-lm-3b-4e1t-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-stable-lm-3b-4e1t-v0"}, + + {"model_name": "llmware/bling-cerebras-1.3b-0.1", "display_name": "bling-cerebras-1.3b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-cerebras-1.3b-0.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-cerebras-1.3b-0.1"}, + + {"model_name": "llmware/bling-tiny-llama-v0", "display_name": "bling-tiny-llama-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-tiny-llama-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-tiny-llama-v0"}, + + # dragon models + {"model_name": "llmware/dragon-yi-6b-v0", "display_name": "dragon-yi-6b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "\n", "link": "https://huggingface.co/llmware/dragon-yi-6b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-yi-6b-v0"}, + + {"model_name": "llmware/dragon-stablelm-7b-v0", "display_name": "dragon-stablelm-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-stablelm-7b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-stablelm-7b-v0"}, + + {"model_name": "llmware/dragon-mistral-7b-v0", "display_name": "dragon-mistral-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-mistral-7b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-mistral-7b-v0"}, + + {"model_name": "llmware/dragon-mistral-0.3", "display_name": "dragon-mistral-0.3", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-mistral-0.3", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-mistral-0.3"}, + + {"model_name": "llmware/dragon-qwen-7b", "display_name": "dragon-qwen-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-qwen-7b", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-qwen-7b"}, + + {"model_name": "llmware/dragon-red-pajama-7b-v0", "display_name": "dragon-red-pajama-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-red-pajama-7b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-red-pajama-7b-v0"}, + + {"model_name": "llmware/dragon-deci-6b-v0", "display_name": "dragon-deci-6b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-deci-6b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-deci-6b-v0"}, + + {"model_name": "llmware/dragon-falcon-7b-v0", "display_name": "dragon-falcon-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-falcon-7b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-falcon-7b-v0"}, + + {"model_name": "llmware/dragon-llama-7b-v0", "display_name": "dragon-llama-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-llama-7b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-llama-7b-v0"}, + + {"model_name": "llmware/dragon-deci-7b-v0", "display_name": "dragon-deci-7b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-deci-7b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-deci-7b-v0"}, + + {"model_name": "llmware/dragon-llama-3.1", "display_name": "dragon-llama-3.1", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/llmware/dragon-llama-3.1", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/dragon-llama-3.1"}, + + {"model_name": "llmware/bling-phi-3", "display_name": "bling-phi-3", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-phi-3", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-phi-3"}, + + {"model_name": "llmware/bling-phi-3.5", "display_name": "bling-phi-3.5", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", "link": "https://huggingface.co/llmware/bling-phi-3.5", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/bling-phi-3.5"}, + + # gguf models + {"model_name": "bling-phi-3-gguf", "display_name": "llmware/bling-phi-3-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "bling-phi-3.gguf", + "gguf_repo": "llmware/bling-phi-3-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-phi-3.gguf"], + "tokenizer_local": "tokenizer_phi3.json", + "link": "https://huggingface.co/llmware/bling-phi-3-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "bling-phi-3.5-gguf", "display_name": "llmware/bling-phi-3.5-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "bling-phi3-5.gguf", + "gguf_repo": "llmware/bling-phi-3.5-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-phi3-5.gguf"], + "tokenizer_local": "tokenizer_phi3.json", + "link": "https://huggingface.co/llmware/bling-phi-3.5-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-llama-3.1-gguf", "display_name": "llmware/dragon-llama-3.1-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "dragon-llama31.gguf", + "gguf_repo": "llmware/dragon-llama-3.1-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-llama31.gguf"], + "tokenizer_local": "tokenizer_phi3.json", + "link": "https://huggingface.co/llmware/dragon-llama-3.1-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-mistral-0.3-gguf", "display_name": "llmware/dragon-mistral-0.3-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "dragon-mistral-03.gguf", + "gguf_repo": "llmware/dragon-mistral-0.3-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-mistral-03.gguf"], + "tokenizer_local": "tokenizer_phi3.json", + "link": "https://huggingface.co/llmware/dragon-mistral-0.3-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "bling-phi-2-gguf", "display_name": "llmware/bling-phi-2-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "bling-phi2-tool.gguf", + "gguf_repo": "llmware/bling-phi-2-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-phi2-tool.gguf"], + "tokenizer_local": "tokenizer_phi2.json", + "link": "https://huggingface.co/llmware/bling-phi-2-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-yi-9b-gguf", "display_name": "llmware/dragon-yi-9b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "dragon-yi-1-5-9.gguf", + "gguf_repo": "llmware/dragon-yi-9b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-yi-1-5-9.gguf"], + "tokenizer_local": "tokenizer_yi.json", + "link": "https://huggingface.co/llmware/dragon-yi-9b-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-qwen-7b-gguf", "display_name": "llmware/dragon-qwen-7b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "dragon-qwen.gguf", + "gguf_repo": "llmware/dragon-qwen-7b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-qwen.gguf"], + "tokenizer_local": "tokenizer_qw.json", + "link": "https://huggingface.co/llmware/dragon-qwen-7b-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "bling-qwen-1.5b-gguf", "display_name": "bling-qwen-mini-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "bling-qwen-1-5b.gguf", + "gguf_repo": "llmware/bling-qwen-mini-tool", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-qwen-1-5b.gguf"], + "tokenizer_local": "tokenizer_qw.json", + "link": "https://huggingface.co/llmware/bling-qwen-1.5b-gguf", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "bling-qwen-0.5b-gguf", "display_name": "llmware/bling-qwen-nano-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "bling-qwen-0-5.gguf", + "gguf_repo": "llmware/bling-qwen-nano-tool", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-qwen-0-5.gguf"], + "tokenizer_local": "tokenizer_qw.json", + "link": "https://huggingface.co/llmware/bling-qwen-nano-tool", + "custom_model_files": [], "custom_model_repo": ""}, + + # deprecated access to dragon-mistral-7b-gguf -> replaced by dragon-mistral-answer-tool + {"model_name": "llmware/dragon-mistral-7b-gguf", "display_name": "dragon-mistral-7b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["dragon-mistral-7b-q4_k_m.gguf"], + "temperature": 0.3, "trailing_space": "", + "gguf_file": "dragon-mistral-7b-q4_k_m.gguf", + "gguf_repo": "llmware/dragon-mistral-7b-v0", + "link": "https://huggingface.co/llmware/dragon-mistral-7b-v0", + "custom_model_files": [], "custom_model_repo": ""}, + + # deprecated access to dragon-llama-7b-gguf -> replaced by dragon-llama-answer-tool + {"model_name": "llmware/dragon-llama-7b-gguf", "display_name": "dragon-llama-7b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "dragon-llama-7b-q4_k_m.gguf", + "gguf_repo": "llmware/dragon-llama-7b-v0", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["dragon-llama-7b-q4_k_m.gguf"], + "link": "https://huggingface.co/llmware/dragon-llama-7b-v0", + "custom_model_files": [], "custom_model_repo": ""}, + + # deprecated access to dragon-yi-6b-gguf -> replaced by dragon-yi-answer-tool + {"model_name": "llmware/dragon-yi-6b-gguf", "display_name": "dragon-yi-6b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "\n", + "gguf_file": "dragon-yi-6b-q4_k_m.gguf", + "gguf_repo": "llmware/dragon-yi-6b-v0", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["dragon-yi-6b-q4_k_m.gguf"], + "link": "https://huggingface.co/llmware/dragon-yi-6b-v0", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-yi-answer-tool", "display_name": "dragon-yi-6b-answer-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "\n", + "gguf_file": "dragon-yi.gguf", + "gguf_repo": "llmware/dragon-yi-answer-tool", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-yi.gguf"], + "link": "https://huggingface.co/llmware/dragon-yi-answer-tool", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-llama-answer-tool", "display_name": "dragon-llama-answer-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "dragon-llama.gguf", + "gguf_repo": "llmware/dragon-llama-answer-tool", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-llama.gguf"], + "link": "https://huggingface.co/llmware/dragon-llama-answer-tool", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dragon-mistral-answer-tool", "display_name": "dragon-mistral-answer-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "dragon-mistral.gguf", + "gguf_repo": "llmware/dragon-mistral-answer-tool", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["dragon-mistral.gguf"], + "link": "https://huggingface.co/llmware/dragon-mistral-answer-tool", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "phi-3.5-gguf", "display_name": "phi-3.5-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "phi_3", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "phi35.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["phi35.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-7B-instruct-gguf", "display_name": "qwen2-7B-instruct-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "qwen2-7b-instruct.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["qwen2-7b-instruct.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-1.5b-instruct-gguf", "display_name": "qwen2-1.5b-instruct-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "qwen-instruct-1-5b.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["qwen-instruct-1-5b.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-0.5b-instruct-gguf", "display_name": "qwen2-0.5b-instruct-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "qwen2-0_5b-instruct-q4_k_m.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["qwen2-0_5b-instruct-q4_k_m.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.1-instruct-gguf", "display_name": "llama-3.1-instruct-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "llama-031-instruct.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["llama-031-instruct.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "TheBloke/Llama-2-7B-Chat-GGUF", "display_name": "llama-2-7b-chat-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "llama-2-7b-chat.Q4_K_M.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["llama-2-7b-chat.Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "TheBloke/OpenHermes-2.5-Mistral-7B-GGUF", "display_name": "openhermes-mistral-7b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": True, "prompt_wrapper": "chat_ml", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "openhermes-2.5-mistral-7b.Q4_K_M.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["openhermes-2.5-mistral-7b.Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "TheBloke/zephyr-7B-beta-GGUF", "display_name": "zephyr-7b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "zephyr-7b-beta.Q4_K_M.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["zephyr-7b-beta.Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "TheBloke/Starling-LM-7B-alpha-GGUF", "display_name": "starling-7b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": True, "prompt_wrapper": "open_chat", + "temperature": 0.3, "trailing_space": "", + "gguf_file": "starling-lm-7b-alpha.Q4_K_M.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["starling-lm-7b-alpha.Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "microsoft/Phi-3-mini-4k-instruct-gguf", "display_name": "phi-3-gguf", "model_family": "GGUFGenerativeModel", + "model_category": "generative_local", "model_location": "llmware_repo", "context_window": 4096, + "instruction_following": False, "prompt_wrapper": "phi_3", "temperature": 0.3, "trailing_space": "", + "gguf_file": "Phi-3-mini-4k-instruct-q4.gguf", + "gguf_repo": "microsoft/Phi-3-mini-4k-instruct-gguf", + "link": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", + "tokenizer_local": "tokenizer_phi3.json", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["Phi-3-mini-4k-instruct-q4.gguf"], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "microsoft/Phi-3-mini-4k-instruct", "display_name": "phi-3", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "microsoft/Phi-3-mini-4k-instruct"}, + + {"model_name": "microsoft/Phi-3-mini-128k-instruct", "display_name": "phi-3-128k", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct-gguf", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "microsoft/Phi-3-mini-128k-instruct"}, + + {"model_name": "Meta-Llama-3-8B-Instruct", "display_name": "llama-3-instruct", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "llama_3_chat", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/meta-llama/Meta-LLama-3-8B-instruct", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "meta-llama/Meta-Llama-3-8B-Instruct"}, + + {"model_name": "Meta-Llama-3-8B", "display_name": "llama-3-base", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "llama_3_chat", + "temperature": 0.3, "trailing_space": "", "link": "https://huggingface.co/meta-llama/Meta-LLama-3-8B", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "meta-llama/Meta-Llama-3-8B"}, + + {"model_name": "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF", "display_name": "llama-3-instruct-qf-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "llama_3_chat", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "Meta-Llama-3-8B-Instruct.Q4_K_M.gguf", + "gguf_repo": "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["Meta-Llama-3-8B-Instruct.Q4_K_M.gguf"], + "link": "https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "QuantFactory/Meta-Llama-3-8B-GGUF", "display_name": "llama-3-base-qf-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "llama_3_chat", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "Meta-Llama-3-8B.Q4_K_M.gguf", + "gguf_repo": "QuantFactory/Meta-Llama-3-8B-GGUF", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["Meta-Llama-3-8B.Q4_K_M.gguf"], + "link": "https://huggingface.co/QuantFactory/Meta-Llama-3-GGUF", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "bartowski/Meta-Llama-3-8B-Instruct-GGUF", "display_name": "llama-3-instruct-bartowski-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "llama_3_chat", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "Meta-Llama-3-8B-Instruct-Q4_K_M.gguf", + "gguf_repo": "bartowski/Meta-Llama-3-8B-Instruct-GGUF", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["Meta-Llama-3-8B-Instruct-Q4_K_M.gguf"], + "link": "https://huggingface.co/bartowski/Meta-Llama-3-8B-Instruct-GGUF", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "tiny-llama-chat-gguf", "display_name": "tiny-llama-chat-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.3, "sample_default": True, "trailing_space": "", + "gguf_file": "tiny-llama-chat.gguf", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["tiny-llama-chat.gguf"], + "link": "https://huggingface.co/llmware/bonchon", + "tokenizer_local": "tokenizer_tl.json", + "custom_model_files": [], "custom_model_repo": ""}, + + # whisper-cpp models + {"model_name": "whisper-cpp-base-english", "display_name": "whisper-en-base", + "model_family": "WhisperCPPModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "ggml-base.en.bin", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["ggml-base.en.bin"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "whisper-cpp-base", "display_name": "whisper-base", + "model_family": "WhisperCPPModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "ggml-base.bin", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["ggml-base.bin"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "whisper-cpp-tiny-diarize", "display_name": "whisper-en-tiny-diarize", + "model_family": "WhisperCPPModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "ggml-small.en-tdrz.bin", + "gguf_repo": "llmware/bonchon", + "fetch": {"module": "llmware.models", "method": "pull_model_from_hf"}, + "validation_files": ["ggml-small.en-trdz.bin"], + "link": "https://huggingface.co/llmware/bonchon", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "slim-ner-tool", "display_name": "slim-ner-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-ner.gguf", + "gguf_repo": "llmware/slim-ner-tool", + "link": "https://huggingface.co/llmware/slim-ner-tool", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-ner.gguf"], + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["people", "location", "organization", "misc"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-sentiment-tool", "display_name": "slim-sentiment-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-sentiment.gguf", + "gguf_repo": "llmware/slim-sentiment-tool", + "link": "https://huggingface.co/llmware/slim-sentiment-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["sentiment"], + "fc_output_values": ["positive", "neutral", "negative"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-sentiment.gguf"]}, + + {"model_name": "slim-emotions-tool", "display_name": "slim-emotions-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-emotions.gguf", + "gguf_repo": "llmware/slim-emotions-tool", + "link": "https://huggingface.co/llmware/slim-emotions-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["emotions"], + "fc_output_values": ["afraid", "anger", "angry", "annoyed", "anticipating", "anxious", "apprehensive", + "ashamed", "caring", "confident", "content", "devastated", "disappointed", "disgusted", + "embarrassed", "excited", "faithful", "fear", "furious", "grateful", "guilty", + "hopeful", "impressed", "jealous", "joy", "joyful", "lonely", "love", "nostalgic", + "prepared", "proud", "sad", "sadness", "sentimental", "surprise", "surprised", + "terrified", "trusting"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-emotions.gguf"]}, + + {"model_name": "slim-ratings-tool", "display_name": "slim-ratings-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-ratings.gguf", + "gguf_repo": "llmware/slim-ratings-tool", + "link": "https://huggingface.co/llmware/slim-ratings-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["rating"], + "fc_output_values": ["1", "2", "3", "4", "5"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-ratings.gguf"]}, + + {"model_name": "slim-intent-tool", "display_name": "slim-intent-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-intent.gguf", + "gguf_repo": "llmware/slim-intent-tool", + "link": "https://huggingface.co/llmware/slim-intent-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["intent"], + "fc_output_values": ["account", "cancel", "complaint", "customer service", "delivery", "feedback", + "invoice", "new account", "order", "payments", "refund", "shipping", + "subscription", "terminate"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-intent.gguf"]}, + + {"model_name": "slim-nli-tool", "display_name": "slim-nli-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-nli.gguf", + "gguf_repo": "llmware/slim-nli-tool", + "link": "https://huggingface.co/llmware/slim-nli-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["evidence"], + "fc_output_values": ["supports", "neutral", "contradicts"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [9996,5924,17821], + "marker_token_lookup": {9996: "contradicts", 5924: "supports", 17821: "neutral"}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-nli.gguf"]}, + + {"model_name": "slim-topics-tool", "display_name": "slim-topics-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-topics.gguf", + "gguf_repo": "llmware/slim-topics-tool", + "link": "https://huggingface.co/llmware/slim-topics-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["topics"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-topics.gguf"]}, + + {"model_name": "slim-tags-tool", "display_name": "slim-tags-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-tags.gguf", "gguf_repo": "llmware/slim-tags-tool", + "link": "https://huggingface.co/llmware/slim-tags-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["tags"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-tags.gguf"]}, + + {"model_name": "slim-sql-tool", "display_name": "slim-sql-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-sql.gguf", + "gguf_repo": "llmware/slim-sql-tool", + "fc_output_values": [], + "link": "https://huggingface.co/llmware/slim-sql-tool", + "custom_model_files": [], "custom_model_repo": "", + "tokenizer": "llmware/slim-sql-1b-v0", + "tokenizer_local": "tokenizer_tl.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-sql.gguf"]}, + + {"model_name": "bling-answer-tool", "display_name": "bling-answer-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "bling-answer.gguf", + "gguf_repo": "llmware/bling-answer-tool", + "link": "https://huggingface.co/llmware/bling-answer-tool", + "custom_model_files": [], "custom_model_repo": "", + "tokenizer": "llmware/bling-tiny-llama-1b-v0", + "tokenizer_local": "tokenizer_tl.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-answer.gguf"]}, + + {"model_name": "slim-category-tool", "display_name": "slim-category-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-category.gguf", + "gguf_repo": "llmware/slim-category-tool", + "link": "https://huggingface.co/llmware/slim-category-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["category"], + "fc_output_values": ["analyst", "announcements", "bonds", "business", "central bank", "commentary", + "commodities", "currencies", "dividend", "earnings", "energy", "entertainment", + "financials", "health", "human resources", "legal and regulation", "macroeconomics", + "markets", "mergers and acquisitions", "opinion", "politics", "public markets", + "science", "sports", "stocks", "tech", "world"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-category.gguf"]}, + + {"model_name": "llmware/slim-intent", "display_name": "slim-intent-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-intent", + "hf_repo": "llmware/slim-intent", + "custom_model_files": [""], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["intent"], + "fc_output_values": ["account", "cancel", "complaint", "customer service", "delivery", "feedback", + "invoice", "new account", "order", "payments", "refund", "shipping", + "subscription", "terminate"], + "function": ["classify"], + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + }, + + {"model_name": "llmware/slim-sentiment", "display_name": "slim-sentiment-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-sentiment", + "hf_repo": "llmware/slim-sentiment", + "custom_model_files": [""], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["sentiment"], + "fc_output_values": ["positive", "neutral", "negative"], + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["classify"]}, + + {"model_name": "llmware/slim-emotions", "display_name": "slim-emotions-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-emotions", + "hf_repo": "llmware/slim-emotions", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["emotions"], + "fc_output_values": ["afraid", "anger", "angry", "annoyed", "anticipating", "anxious", "apprehensive", + "ashamed", "caring", "confident", "content", "devastated", "disappointed", "disgusted", + "embarrassed", "excited", "faithful", "fear", "furious", "grateful", "guilty", + "hopeful", "impressed", "jealous", "joy", "joyful", "lonely", "love", "nostalgic", + "prepared", "proud", "sad", "sadness", "sentimental", "surprise", "surprised", + "terrified", "trusting"], + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["classify"]}, + + {"model_name": "llmware/slim-ner", "display_name": "slim-ner-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-ner", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "hf_repo": "llmware/slim-ner", + "function_call": True, + "primary_keys": ["person", "organization", "place", "misc"], + "fc_output_values": [], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "llmware/slim-nli", "display_name": "slim-nli-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-nli", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/slim-nli", + "output_type": "dict", + "function_call": True, + "primary_keys": ["evidence"], + "fc_output_values": ["supports", "neutral", "contradicts"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "llmware/slim-ratings", "display_name": "slim-ratings-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-ratings", + "hf_repo": "llmware/slim-ratings", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["rating"], + "fc_output_values": ["1", "2", "3", "4", "5"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "llmware/slim-category", "display_name": "slim-category-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-category", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "hf_repo": "llmware/slim-category", + "function_call": True, + "primary_keys": ["category"], + "fc_output_values": ["analyst", "announcements", "bonds", "business", "central bank", "commentary", + "commodities", "currencies", "dividend", "earnings", "energy", "entertainment", + "financials", "health", "human resources", "legal and regulation", "macroeconomics", + "markets", "mergers and acquisitions", "opinion", "politics", "public markets", + "science", "sports", "stocks", "tech", "world"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "llmware/slim-tags", "display_name": "slim-tags-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-tags", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/slim-tags", + "outout_type": "dict", + "function_call": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "primary_keys": ["tags"], + "fc_output_values": [], + "function": ["classify"]}, + + {"model_name": "llmware/slim-topics", "display_name": "slim-topics-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0,"sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-topics", + "hf_repo": "llmware/slim-topics", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "primary_keys": ["topics"], + "fc_output_values": [], + "function": ["classify"]}, + + {"model_name": "llmware/slim-sql-1b-v0", "display_name": "slim-sql-1b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, + "trailing_space": "", "link": "https://huggingface.co/llmware/slim-sql-1b-v0", + "custom_model_files": [], "custom_model_repo": "", + "hf_repo": "llmware/slim-sql-1b-v0", + #TODO: assess how to handle SQL models with function call parameters + "function_call": False, + "fc_output_values": [], + "primary_keys": ["sql"], "function": ["sql"]}, + + {"model_name": "bling-stablelm-3b-tool", "display_name": "llmware/bling-stablelm-3b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "bling-stablelm.gguf", + "gguf_repo": "llmware/bling-stablelm-3b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["bling-stablelm.gguf"], + "link": "https://huggingface.co/llmware/bling-stablelm-3b-gguf", + "tokenizer_local": "tokenizer_stablelm.json", + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "slim-xsum", "display_name": "llmware/slim-xsum", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-xsum", "hf_repo": "llmware/slim-xsum", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, "primary_keys": ["xsum"], "fc_output_values": [], + "function": ["classify"]}, + + {"model_name": "slim-xsum-tool", "display_name": "slim-xsum-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-xsum.gguf", "gguf_repo": "llmware/slim-xsum-tool", + "link": "https://huggingface.co/llmware/slim-xsum-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", "function_call": True, "primary_keys": ["xsum"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract", + "tokenizer_local": "tokenizer_stablelm.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-xsum.gguf"], + }, + + {"model_name": "slim-extract", "display_name": "llmware/slim-extract", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-extract", "hf_repo": "llmware/slim-extract", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, "primary_keys": ["key data points"], "fc_output_values": [], + "function": ["extract"]}, + + {"model_name": "slim-extract-tiny", "display_name": "llmware/slim-extract-tiny", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-extract-tiny", "hf_repo": "llmware/slim-extract-tiny", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, "primary_keys": ["key data points"], "fc_output_values": [], + "function": ["extract"]}, + + {"model_name": "slim-extract-tool", "display_name": "slim-extract-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-extract.gguf", "gguf_repo": "llmware/slim-extract-tool", + "link": "https://huggingface.co/llmware/slim-extract-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key data points"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract", + "tokenizer_local": "tokenizer_stablelm.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["extract"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-extract.gguf"], + }, + + {"model_name": "slim-extract-phi-3-gguf", "display_name": "slim-extract-phi-3-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "phi3-extract.gguf", "gguf_repo": "llmware/slim-extract-phi-3-gguf", + "link": "https://huggingface.co/llmware/slim-extract-phi-3-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key data points"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["extract"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["phi3-extract.gguf"], + }, + + {"model_name": "slim-extract-qwen-1.5b-gguf", "display_name": "slim-extract-qwen-1.5b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "qwen-extract-1-5.gguf", "gguf_repo": "llmware/slim-extract-qwen-1.5b-gguf", + "link": "https://huggingface.co/llmware/slim-extract-qwen-1.5b-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key data points"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract-qwen-1.5b", + "tokenizer_local": "tokenizer_qw.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["extract"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qwen-extract-1-5.gguf"], + }, + + {"model_name": "slim-extract-qwen-nano-gguf", "display_name": "slim-extract-qwen-0.5b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "qwen-extract-0-5.gguf", "gguf_repo": "llmware/slim-extract-qwen-0.5b-gguf", + "link": "https://huggingface.co/llmware/slim-extract-qwen-0.5b-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key data points"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract-qwen-0.5b-gguf", + "tokenizer_local": "tokenizer_qw.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["extract"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qwen-extract-0-5.gguf"], + }, + + {"model_name": "llmware/slim-extract-tiny-tool", "display_name": "slim-extract-tiny-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot","temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "tiny-extract.gguf", "gguf_repo": "llmware/slim-extract-tiny-tool", + "link": "https://huggingface.co/llmware/slim-extract-tiny-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key points"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["tiny-extract.gguf"]}, + + {"model_name": "llmware/slim-summary-tiny-tool", "display_name": "slim-summary-tiny-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "tiny-summary.gguf", "gguf_repo": "llmware/slim-summary-tiny-tool", + "link": "https://huggingface.co/llmware/slim-summary-tiny-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True,"primary_keys": ["key points"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["tiny-summary.gguf"]}, + + {"model_name": "slim-summary-phi-3-gguf", "display_name": "slim-summary-phi-3-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "phi3-summary.gguf", "gguf_repo": "llmware/slim-summary-phi-3-gguf", + "link": "https://huggingface.co/llmware/slim-summary-phi-3-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key points"], "fc_output_values": [], + "tokenizer": "llmware/slim-summary-phi3", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["phi3-summary.gguf"]}, + + {"model_name": "slim-xsum-phi-3-gguf", "display_name": "slim-xsum-phi-3-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-xsum.gguf", "gguf_repo": "llmware/slim-xsum-phi-3-gguf", + "link": "https://huggingface.co/llmware/slim-xsum-phi-3-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["key points"], "fc_output_values": [], + "tokenizer": "llmware/slim-xsum-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-xsum.gguf"]}, + + {"model_name": "slim-boolean", "display_name": "llmware/slim-boolean", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-boolean", "hf_repo": "llmware/slim-boolean", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", "function_call": True, + "marker_tokens": [2369,9820], "marker_token_lookup": {2369: "no", 9820: "yes"}, + "primary_keys": [], "fc_output_values": [], + "function": ["boolean"]}, + + {"model_name": "slim-boolean-tool", "display_name": "slim-boolean-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-boolean.gguf", "gguf_repo": "llmware/slim-boolean-tool", + "link": "https://huggingface.co/llmware/slim-boolean-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": [], "fc_output_values": [], + "tokenizer": "llmware/slim-extract", + "tokenizer_local": "tokenizer_stablelm.json", + "marker_tokens": [2369,9820], "marker_token_lookup": {2369: "no", 9820: "yes"}, + "function": ["boolean"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-boolean.gguf"], + }, + + {"model_name": "slim-boolean-phi-3-gguf", "display_name": "slim-boolean-phi-3-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-boolean.gguf", "gguf_repo": "llmware/slim-boolean-phi-3-gguf", + "link": "https://huggingface.co/llmware/slim-boolean-phi-3-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": [], "fc_output_values": [], + "tokenizer": "llmware/slim-boolean-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [2369, 9820], "marker_token_lookup": {2369: "no", 9820: "yes"}, + "function": ["boolean"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-boolean.gguf"], + }, + + {"model_name": "slim-sa-ner", "display_name": "llmware/slim-sa-ner", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-sa-ner", "hf_repo": "llmware/slim-sa-ner", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, + "primary_keys": ["sentiment, person, organization, place"], "fc_output_values": [], + "function": ["classify"]}, + + {"model_name": "slim-sa-ner-phi-3-gguf", "display_name": "slim-sa-ner-phi-3-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-sa-ner.gguf", "gguf_repo": "llmware/slim-sa-ner-phi-3-gguf", + "link": "https://huggingface.co/llmware/slim-sa-ner-phi-3-gguf", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["sentiment, person, organization, place"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-sa-ner.gguf"], + }, + + {"model_name": "slim-sa-ner-tool", "display_name": "slim-sa-ner-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "sa-ner.gguf", "gguf_repo": "llmware/slim-sa-ner-tool", + "link": "https://huggingface.co/llmware/slim-sa-ner-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["sentiment, person, organization, place"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract", + "tokenizer_local": "tokenizer_stablelm.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["sa-ner.gguf"], + }, + + {"model_name": "slim-tags-3b", "display_name": "llmware/slim-tags-3b", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-tags-3b", "hf_repo": "llmware/slim-tags-3b", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, + "primary_keys": ["tags"], "fc_output_values": [], + "function": ["classify"]}, + + {"model_name": "slim-tags-3b-tool", "display_name": "slim-tags-3b-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-tags-3b.gguf", "gguf_repo": "llmware/slim-tags-3b-tool", + "link": "https://huggingface.co/llmware/slim-tags-3b-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["tags"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract", + "tokenizer_local": "tokenizer_stablelm.json", + "marker_tokens": [], + "marker_token_lookup": {}, "function": ["classify"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-tags-3b.gguf"], + }, + + {"model_name": "slim-summary", "display_name": "llmware/slim-summary", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-summary", "hf_repo": "llmware/slim-summary", + "custom_model_files": [], "custom_model_repo": "", "output_type": "list", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, "primary_keys": ["key points (3)"], "fc_output_values": [], + "function": ["summarize"]}, + + {"model_name": "slim-summary-tiny", "display_name": "llmware/slim-summary-tiny", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-summary-tiny", "hf_repo": "llmware/slim-summary-tiny", + "custom_model_files": [], "custom_model_repo": "", "output_type": "list", "function_call": True, + "marker_tokens": [], "marker_token_lookup": {}, "primary_keys": ["key points (3)"], "fc_output_values": [], + "function": ["summarize"]}, + + {"model_name": "slim-summary-tool", "display_name": "slim-summary-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "slim-summarize.gguf", "gguf_repo": "llmware/slim-summary-tool", + "link": "https://huggingface.co/llmware/slim-summary-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "list", + "function_call": True, "primary_keys": ["key points (3)"], "fc_output_values": [], + "tokenizer": "llmware/slim-extract", + "tokenizer_local": "tokenizer_stablelm.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["summarize"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["slim-summarize.gguf"], + }, + + {"model_name": "slim-q-gen-phi-3-tool", "display_name": "slim-q-gen-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "sample_default": True, "trailing_space": "", + "gguf_file": "q_gen.gguf", + "gguf_repo": "llmware/slim-q-gen-phi-3-tool", + "link": "https://huggingface.co/llmware/slim-q-gen-phi-3-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["question"], + "fc_output_values": [], + "tokenizer": "microsoft/Phi-3-mini-4k-instruct", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["generate"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["q_gen.gguf"]}, + + {"model_name": "slim-q-gen-tiny-tool", "display_name": "llmware/slim-q-gen-tiny-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.5, "sample_default": True, "trailing_space": "", + "gguf_file": "q_gen.gguf", + "gguf_repo": "llmware/slim-q-gen-tiny-tool", + "link": "https://huggingface.co/slim-q-gen-tiny-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["question"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["generate"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["q_gen.gguf"], + }, + + {"model_name": "llmware/slim-q-gen-tiny", "display_name": "slim-q-gen-tiny", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.5, "sample_default": True, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-q-gen-tiny", + "hf_repo": "llmware/slim-q-gen-tiny", + "custom_model_files": [""], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["question"], + "fc_output_values": ["question"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["generate"]}, + + {"model_name": "llmware/slim-q-gen-phi-3", "display_name": "slim-q-gen-phi-3", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.5, "sample_default": True, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-q-gen-phi-3", + "hf_repo": "llmware/slim-q-gen-phi-3", + "custom_model_files": [""], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["question"], + "fc_output_values": ["question"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["generate"]}, + + {"model_name": "slim-qa-gen-tiny-tool", "display_name": "llmware/slim-qa-gen-tiny-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.5, "sample_default": True, "trailing_space": "", + "gguf_file": "qa_gen_v3.gguf", + "gguf_repo": "llmware/slim-qa-gen-tiny-tool", + "link": "https://huggingface.co/slim-qa-gen-tiny-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["question, answer"], # also accepts boolean and multiple choice + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["generate"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qa_gen_v3.gguf"], + }, + + {"model_name": "slim-qa-gen-phi-3-tool", "display_name": "slim-qa-gen-phi-3-tool", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.3, "sample_default": True, "trailing_space": "", + "gguf_file": "qa_gen_v3.gguf", + "gguf_repo": "llmware/slim-qa-gen-phi-3-tool", + "link": "https://huggingface.co/llmware/slim-qa-gen-phi-3-tool", + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", + "function_call": True, + "primary_keys": ["question, answer"], # also accepts boolean and multiple choice + "fc_output_values": [], + "tokenizer": "microsoft/Phi-3-mini-4k-instruct", + "tokenizer_local": "tokenizer_phi3.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["generate"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qa_gen_v3.gguf"]}, + + {"model_name": "llmware/slim-qa-gen-tiny", "display_name": "slim-qa-gen-tiny", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.5, "sample_default": True, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-qa-gen-tiny", + "hf_repo": "llmware/slim-qa-gen-tiny", + "custom_model_files": [""], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["question, answer"], + "fc_output_values": ["question, answer"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["generate"]}, + + {"model_name": "llmware/slim-qa-gen-phi-3", "display_name": "slim-qa-gen-phi-3", + "model_family": "HFGenerativeModel", "model_category": "generative_local", "model_location": "hf_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.5, "sample_default": True, "trailing_space": "", "gguf_file": "", "gguf_repo": "", + "link": "https://huggingface.co/llmware/slim-qa-gen-phi-3", + "hf_repo": "llmware/slim-qa-gen-phi-3", + "custom_model_files": [""], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["question, answer"], + "fc_output_values": ["question, answer"], + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["generate"]}, + + {"model_name": "bling-qwen-500m-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "bling-qwen-500m-ov", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/bling-qwen-500m-ov", + "tokenizer_local": "tokenizer_qw.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/bling-qwen-500m-ov"}, + + {"model_name": "bling-qwen-1.5b-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "bling-qwen-1.5b-ov", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/bling-qwen-1.5b-ov", + "tokenizer_local": "tokenizer_qw.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/bling-qwen-1.5b-ov"}, + + {"model_name": "dragon-qwen-7b-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-qwen-7b-ov", + "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-qwen-7b-ov", + "tokenizer_local": "tokenizer_qw.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-qwen-7b-ov"}, + + {"model_name": "slim-xsum-phi-3-ov", "display_name": "slim-xsum-phi-3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-xsum-phi-3-ov", + "link": "https://huggingface.co/llmware/slim-xsum-phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["xsum"], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["generate"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-boolean-phi-3-ov", "display_name": "slim-boolean-phi-3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-boolean-phi-3-ov", + "link": "https://huggingface.co/llmware/slim-boolean-phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": [""], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["boolean"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-sa-ner-phi-3-ov", "display_name": "slim-sa-ner-phi-3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-sa-ner-phi-3-ov", + "link": "https://huggingface.co/llmware/slim-sa-ner-phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["sentiment", "people"], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["classify"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-summary-phi-3-ov", "display_name": "slim-summary-phi-3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-summary-phi-3-ov", + "link": "https://huggingface.co/llmware/slim-summary-phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["summarize"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-extract-qwen-0.5b-ov", "display_name": "slim-extract-qwen-0.5b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-extract-qwen-0.5b-ov", + "link": "https://huggingface.co/llmware/slim-extract-qwen-0.5b-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/slim-qwen-extract-500m", + "tokenizer_local": "tokenizer_qw.json", + "function": ["extract"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-extract-qwen-1.5b-ov", "display_name": "slim-extract-qwen-1.5b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-extract-qwen-1.5b-ov", + "link": "https://huggingface.co/llmware/slim-extract-qwen-1.5b-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_qw.json", + "function": ["extract"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-summary-tiny-ov", "display_name": "slim-summary-tiny-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_repo": "llmware/slim-summary-tiny-ov", + "hf_repo": "llmware/slim-summary-tiny-ov", + "link": "https://huggingface.co/llmware/slim-summary-tiny-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "list", + "function_call": True, "primary_keys": ["key points (3)"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["summarize"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-sql-ov", "display_name": "slim-sql-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-sql-ov", + "fc_output_values": [], "link": "https://huggingface.co/llmware/slim-sql-ov", + "custom_model_files": [], "custom_model_repo": "", "tokenizer": "llmware/slim-sql-1b-v0", + "tokenizer_local": "tokenizer_tl.json", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-emotions-ov", "display_name": "slim-emotions-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-emotions-ov", + "link": "https://huggingface.co/llmware/slim-emotions-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["emotions"], + "fc_output_values": ["afraid", "anger", "angry", "annoyed", "anticipating", "anxious", "apprehensive", + "ashamed", "caring", "confident", "content", "devastated", "disappointed", "disgusted", + "embarrassed", "excited", "faithful", "fear", "furious", "grateful", "guilty", + "hopeful", "impressed", "jealous", "joy", "joyful", "lonely", "love", "nostalgic", + "prepared", "proud", "sad", "sadness", "sentimental", "surprise", "surprised", + "terrified", "trusting"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-topics-ov", "display_name": "slim-topics-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-topics-ov", + "link": "https://huggingface.co/llmware/slim-topics-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["topics"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-ner-ov", "display_name": "slim-ner-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-ner-ov", + "link": "https://huggingface.co/llmware/slim-ner-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["people", "location", "organization", "misc"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-intent-ov", "display_name": "slim-intent-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-intent-ov", + "link": "https://huggingface.co/llmware/slim-intent-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "custom_model_files":[], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["intent"], + "fc_output_Values": [], + "tokenizer": "llmware/slim-intent", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-tags-ov", "display_name": "slim-tags-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-tags-ov", + "link": "https://huggingface.co/llmware/slim-tags-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "custom_model_files":[], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["tags"], + "fc_output_Values": [], + "tokenizer": "llmware/slim-tags", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-ratings-ov", "display_name": "slim-ratings-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-ratings-ov", + "link": "https://huggingface.co/llmware/slim-ratings-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "custom_model_files":[], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["ratings"], + "fc_output_Values": [], + "tokenizer": "llmware/slim-ratings", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "dragon-mistral-0.3-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-mistral-0.3-ov", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-mistral-0.3-ov", + "tokenizer_local": "tokenizer_mistral.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-mistral-0.3-ov"}, + + {"model_name": "dragon-yi-6b-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-yi-6b-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-yi-6b-ov", + "tokenizer_local": "tokenizer_yi.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-yi-6b-ov"}, + + {"model_name": "dragon-yi-9b-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "dragon-yi-9b-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-yi-9b-ov", + "tokenizer_local": "tokenizer_yi.json", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "link": "https://huggingface.co/llmware/dragon-yi-9b-ov"}, + + {"model_name": "llama-2-chat-ov", "display_name": "llama-2-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/llama-2-chat-ov", + "link": "https://huggingface.co/llmware/llama-2-chat-ov", + "tokenizer_local": "tokenizer_ll2.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-2-13b-chat-ov", "display_name": "llama-2-13b-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/llama-2-13b-chat-ov", + "link": "https://huggingface.co/llmware/llama-2-13b-chat-ov", + "tokenizer_local": "tokenizer_ll2.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "tiny-llama-chat-ov", "display_name": "tiny-llama-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "tiny_llama_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/tiny-llama-chat-ov", + "link": "https://huggingface.co/llmware/tiny-llama-chat-ov", + "tokenizer_local": "tokenizer_tl.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-7b-instruct-ov", "display_name": "qwen2-7b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "\n", + "hf_repo": "llmware/qwen2-7b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2-7b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "yi-9b-chat-ov", "display_name": "yi-9b-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "\n", + "hf_repo": "llmware/yi-9b-chat-ov", + "link": "https://huggingface.co/llmware/yi-9b-chat-ov", + "tokenizer_local": "tokenizer_yi.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-7b-instruct-v0.3-ov", "display_name": "mistral-7b-instruct-v0.3-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-7b-instruct-v0.3-ov", + "link": "https://huggingface.co/llmware/mistral-7b-instruct-v0.3-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-small-instruct-2409-ov", "display_name": "mistral-small-instruct-2409-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-small-instruct-2409-ov", + "link": "https://huggingface.co/llmware/mistral-small-instruct-2409-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-nemo-instruct-2407-ov", "display_name": "mistral-nemo-instruct-2407-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-nemo-instruct-2407-ov", + "link": "https://huggingface.co/llmware/mistral-nemo-instruct-2407-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-7b-instruct-v0.2-ov", "display_name": "mistral-7b-instruct-v0.2-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-7b-instruct-v0.2-ov", + "link": "https://huggingface.co/llmware/mistral-7b-instruct-v0.2-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "zephyr-mistral-7b-chat-ov", "display_name": "zephyr-mistral-7b-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/zephyr-mistral-7b-chat-ov", + "link": "https://huggingface.co/llmware/zephyr-mistral-7b-chat-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "gemma-7b-it-ov", "display_name": "gemma-7b-it-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "google_gemma_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/gemma-7b-it-ov", + "link": "https://huggingface.co/llmware/gemma-7b-it-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "tokenizer_local": "tokenizer_gemma.json", + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "codegemma-7b-it-ov", "display_name": "codegemma-7b-it-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "google_gemma_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/codegemma-7b-it-ov", + "link": "https://huggingface.co/llmware/codegemma-7b-it-ov", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "gemma-2b-it-ov", "display_name": "gemma-2b-it-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "google_gemma_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/gemma-2b-it-ov", + "link": "https://huggingface.co/llmware/gemma-2b-it-ov", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "stablelm-zephyr-3b-ov", "display_name": "stablelm-zephyr-3b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "stablelm_zephyr_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/stablelm-zephyr-3b-ov", + "link": "https://huggingface.co/llmware/stablelm-zephyr-3b-ov", + "tokenizer_local": "tokenizer_stablelm.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "teknium-open-hermes-2.5-mistral-ov", "display_name": "teknium-open-hermes-2.5-mistral-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/teknium-open-hermes-2.5-mistral-ov", + "link": "https://huggingface.co/llmware/teknium-open-hermes-2.5-mistral-ov", + "tokenizer_local": "tokenizer_mistral_chat.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.1-instruct-ov", "display_name": "llama-3.1-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/llama-3.1-instruct-ov", + "link": "https://huggingface.co/llmware/llama-3.1-instruct-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-1.5b-instruct-ov", "display_name": "qwen2-1.5b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2-1.5b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2-1.5b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-0.5b-chat-ov", "display_name": "qwen2-0.5b-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2-0.5b-chat-ov", + "link": "https://huggingface.co/llmware/qwen2-0.5b-chat-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "nvidia-llama3-chatqa-1.5-8b-ov", "display_name": "nvidia-llama3-chatqa-1.5-8b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/nvidia-llama3-chatqa-1.5-8b-ov", + "link": "https://huggingface.co/llmware/nvidia-llama3-chatqa-1.5-8b-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "yi-6b-1.5v-chat-ov", "display_name": "yi-6b-1.5v-chat-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/yi-6b-1.5v-chat-ov", + "link": "https://huggingface.co/llmware/yi-6b-1.5v-chat-ov", + "tokenizer_local": "tokenizer_yi.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "tiny-dolphin-2.8-1.1b-ov", "display_name": "tiny-dolphin-2.8b-1.1b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/tiny-dolphin-2.8-1.1b-ov", + "link": "https://huggingface.co/llmware/tiny-dolphin-2.8-1.1b-ov", + "tokenizer_local": "tokenizer_tl.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": "" + }, + + {"model_name": "dolphin-2.9.3-mistral-7b-32k-ov", "display_name": "dolphin-2.9.3-mistral-7b-32k-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/dolphin-2.9.3-mistral-7b-32k-ov", + "link": "https://huggingface.co/llmware/dolphin-2.9.3-mistral-7b-32k-ov", + "tokenizer_local": "tokenizer_mistral_chat.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": "" + }, + + {"model_name": "dolphin-2.9.4-llama3.1-8b-ov", "display_name": "dolphin-2.9.4-llama3.1-8b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/dolphin-2.9.4-llama3.1-8b-ov", + "link": "https://huggingface.co/llmware/dolphi-2.9.4-llama3.1-8b-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": "" + }, + + {"model_name": "intel-neural-chat-7b-v3-2-ov", "display_name": "intel-neural-chat-7b-v3-2-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/intel-neural-chat-7b-v3-2-ov", + "link": "https://huggingface.co/llmware/intel-neural-chat-7b-v3-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": "" + }, + + {"model_name": "stablelm-2-zephyr-1_6b-ov", "display_name": "stablelm-2-zephyr-1_6b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "stablelm_zephyr_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/stablelm-2-zephyr-1_6b-ov", + "link": "https://huggingface.co/llmware/stablelm-2-zephyr-1_6b-ov", + "tokenizer_local": "tokenizer_stablelm_1_6.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "dreamgen-wizardlm-2-7b-ov", "display_name": "dreamgen-wizardlm-2-7b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/dreamgen-wizardlm-2-7b-ov", + "link": "https://huggingface.co/llmware/dreamgen-wizardlm-2-7b-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "openchat-3.6-8b-20240522-ov", "display_name": "llmware/openchat-3.6-8b-20240522-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/openchat-3.6-8b-20240522-ov", + "link": "https://huggingface.co/llmware/openchat-3.6-8b-20240522-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mathstral-7b-ov", "display_name": "mathstral-7b-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mathstral-7b-ov", + "link": "https://huggingface.co/llmware/mathstral-7b-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-0.5b-instruct-ov", "display_name": "qwen2.5-0.5b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2-0.5b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-0.5b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-3b-instruct-ov", "display_name": "qwen2.5-3b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-3b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-3b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-7b-instruct-ov", "display_name": "qwen2.5-7b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-7b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-7b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-14b-instruct-ov", "display_name": "qwen2.5-14b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-14b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-14b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3-8b-ov", "display_name": "qwen3-8b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 32768, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/qwen3-8b-ov", + "link": "https://huggingface.co/llmware/qwen3-8b-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 8, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3-1.7b-ov", "display_name": "qwen3-1.7b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 32768, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/qwen3-1.7b-ov", + "link": "https://huggingface.co/llmware/qwen3-1.7b-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 1.7, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3-4b-ov", "display_name": "qwen3-4b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 262144, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/qwen3-4b-ov", + "link": "https://huggingface.co/llmware/qwen3-4b-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 4, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3-14b-ov", "display_name": "qwen3-14b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 32768, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/qwen3-14b-ov", + "link": "https://huggingface.co/llmware/qwen3-14b-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 14, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-32b-instruct-ov", "display_name": "qwen-2.5-32b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-32b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-32b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 32.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-72b-instruct-ov", "display_name": "qwen-2.5-72b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-72b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-72b-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 72.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "granite-4-micro-ov", "display_name": "granite-4-3b", + "model_family": "OVGenerativeModel", + "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "granite_chat", + "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/granite-4-micro-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "tokenizer_local": "tokenizer_granite.json", + "link": "https://huggingface.co/llmware/granite-4-micro-ov", + "custom_model_files": [], "custom_model_repo": "", "parameters": 2.6}, + + {"model_name": "phi-4-ov", "display_name": "phi-4-14b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_windows": 4096, "instruction_following": False, + "prompt_wrapper": "phi_4", "temperature": 0.0, "trailing_space": "", + "parameters": 14.0, + "hf_repo": "llmware/phi-4-ov", + "link": "https://huggingface.co/llmware/phi-4-ov", + "tokenizer_local": "tokenizer_phi4.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "phi-4-mini-ov", "display_name": "phi-4-mini-4b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_windows": 4096, "instruction_following": False, + "prompt_wrapper": "phi_4_mini", "temperature": 0.0, "trailing_space": "", + "parameters": 4.0, + "hf_repo": "llmware/phi-4-mini-ov", + "link": "https://huggingface.co/llmware/phi-4-mini-ov", + "tokenizer_local": "tokenizer_phi4_mini.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "phi-4-npu-ov", "display_name": "phi-4-npu-14b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_windows": 4096, "instruction_following": False, + "prompt_wrapper": "phi_4", "temperature": 0.0, "trailing_space": "", + "parameters": 14.0, + "hf_repo": "llmware/phi-4-npu-ov", + "link": "https://huggingface.co/llmware/phi-4-npu-ov", + "tokenizer_local": "tokenizer_phi4.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "npu_optimized": True, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.2-3b-instruct-npu-ov", "display_name": "llama-3.2-npu-3b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/llama-3.2-3b-instruct-npu-ov", + "link": "https://huggingface.co/llmware/llama-3.2-3b-instruct-npu-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 3.0, + "npu_optimized": True, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "bling-tiny-llama-npu-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "bling-tiny-llama-npu-1b", + "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_tl.json", + "hf_repo": "llmware/bling-tiny-llama-npu-v2-ov", "parameters": 1.1, + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "npu_optimized": True, + "link": "https://huggingface.co/llmware/bling-tiny-llama-npu-v2-ov"}, + + {"model_name": "slim-sentiment-npu-ov", "display_name": "agent-sentiment", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "gguf_file": "slim-sentiment.gguf", + "hf_repo": "llmware/slim-sentiment-npu-ov", "parameters": 1.1, + "link": "https://huggingface.co/llmware/slim-sentiment-npu-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "npu_optimized": True, + "primary_keys": ["sentiment"], + "fc_output_values": ["positive", "neutral", "negative"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-extract-tiny-npu-ov", "display_name": "agent-extract-tiny-npu", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-extract-tiny-npu-ov", + "link": "https://huggingface.co/llmware/slim-extract-tiny-npu-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "parameters": 1.1, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "function": ["extract"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, "npu_optimized": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-sql-npu-ov", "display_name": "agent-npu-sql", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-sql-npu-ov", + "fc_output_values": [], "link": "https://huggingface.co/llmware/slim-sql-npu-ov", + "custom_model_files": [], "custom_model_repo": "", "tokenizer": "llmware/slim-sql-1b-v0", + "tokenizer_local": "tokenizer_tl.json", + "snapshot": True, "parameters": 1.1, "npu_optimized": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-emotions-npu-ov", "display_name": "agent-emotions", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "", "gguf_repo": "llmware/slim-emotions-npu-ov", + "hf_repo": "llmware/slim-emotions-npu-ov", "parameters": 1.1, + "link": "https://huggingface.co/llmware/slim-emotions-npu-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["emotions"], + "fc_output_values": ["afraid", "anger", "angry", "annoyed", "anticipating", "anxious", "apprehensive", + "ashamed", "caring", "confident", "content", "devastated", "disappointed", "disgusted", + "embarrassed", "excited", "faithful", "fear", "furious", "grateful", "guilty", + "hopeful", "impressed", "jealous", "joy", "joyful", "lonely", "love", "nostalgic", + "prepared", "proud", "sad", "sadness", "sentimental", "surprise", "surprised", + "terrified", "trusting"], + "tokenizer": "llmware/slim-sentiment", "npu_optimized": True, + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"]}, + + {"model_name": "slim-ratings-npu-ov", "display_name": "agent-npu-ratings", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-ratings-npu-ov", + "link": "https://huggingface.co/llmware/slim-ratings-npu-ov", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"], + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["ratings"], "parameters": 1.1, + "fc_output_Values": [], + "tokenizer": "llmware/slim-ratings-npu", "npu_optimized": True, + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "phi-3-npu-ov", "display_name": "phi-3-npu-3b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_windows": 4096, "instruction_following": False, + "prompt_wrapper": "phi_3", "temperature": 0.3, "trailing_space": "", + "npu_optimized": True, + "hf_repo": "llmware/phi-3-npu-ov", + "link": "https://huggingface.co/llmware/phi-3-npu-ov", + "tokenizer_local": "tokenizer_phi3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 3.8, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "yi-9b-npu-ov", "display_name": "yi-9b-npu", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "chat_ml", "temperature": 0.3, "trailing_space": "\n", + "hf_repo": "llmware/yi-9b-npu-ov", + "link": "https://huggingface.co/llmware/yi-9b-npu-ov", + "tokenizer_local": "tokenizer_yi.json", "npu_optimized": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], "parameters": 8.8, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-7b-v0.3-npu-ov", "display_name": "mistral-0.3-npu-7b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-7b-v0.3-npu-ov", + "link": "https://huggingface.co/llmware/mistral-7b-v0.3-npu-ov", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], + "parameters": 7.3, "npu_optimized": True, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.1-8b-instruct-npu-ov", "display_name": "llama-3.1-npu-8b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/llama-3.1-8b-instruct-npu-ov", + "link": "https://huggingface.co/llmware/llama-3.1-8b-instruct-npu-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml", "openvino_model.bin"], + "parameters": 8.0, "npu_optimized": True, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "slim-tags-npu-ov", "display_name": "agent-npu-tags", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "gguf_file": "", "gguf_repo": "llmware/slim-tags-npu-ov", + "link": "https://huggingface.co/llmware/slim-tags-npu-ov", + "fetch": {"module": "llmware.models", + "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["tags"], + "fc_output_Values": [], "parameters": 1.1, + "tokenizer": "llmware/slim-tags", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"], "npu_optimized": True + }, + + {"model_name": "slim-topics-npu-ov", "display_name": "agent-topics", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "gguf_file": "", "gguf_repo": "llmware/slim-topics-npu-ov", + "hf_repo": "llmware/slim-topics-npu-ov", "parameters": 1.1, + "link": "https://huggingface.co/llmware/slim-topics-npu-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["topics"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["classify"], + "snapshot": True, "npu_optimized": True, + "fetch": {"snapshot": True, "module": "llmware.models", + "method": "pull_snapshot_from_hf"}, + "validation_files": [], + }, + + {"model_name": "llama-3.2-1b-instruct-npu-ov", "display_name": "llama-3.2-npu-1b", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/llama-3.2-1b-instruct-npu-ov", + "link": "https://huggingface.co/llmware/llama-3.2-1b-npu-instruct-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 1.1, + "custom_model_files": [], "custom_model_repo": "", + "npu_optimized": True}, + + {"model_name": "llama-3.2-3b-instruct-ov", "display_name": "llama-3.2-3b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "gguf_repo": "llmware/llama-3.2-3b-instruct-ov", + "link": "https://huggingface.co/llmware/llama-3.2-3b-instruct-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.2-1b-instruct-ov", "display_name": "llama-3.2-1b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "gguf_repo": "llmware/llama-3.2-1b-instruct-ov", + "link": "https://huggingface.co/llmware/llama-3.2-1b-instruct-ov", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-coder-7b-instruct-ov", "display_name": "qwen2.5-coder-7b-instruct-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-coder-7b-instruct-ov", + "link": "https://huggingface.co/llmware/qwen2.5-7b-coder-instruct-ov", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "slim-q-gen-tiny-ov", "display_name": "slim-q-gen-tiny-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-q-gen-tiny-ov", + "link": "https://huggingface.co/llmware/slim-q-gen-tiny-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["question"], + "fc_output_values": ["question"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-qa-gen-tiny-ov", "display_name": "slim-qa-gen-tiny-ov", + "model_family": "OVGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-qa-gen-tiny-ov", + "link": "https://huggingface.co/llmware/slim-qa-gen-tiny-ov", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["question, answer"], + "fc_output_values": ["question"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.bin", "openvino_model.xml"] + }, + + {"model_name": "slim-sentiment-onnx", "display_name": "slim-sentiment-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-sentiment-onnx", + "link": "https://huggingface.co/llmware/slim-sentiment-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["sentiment"], + "fc_output_values": ["positive", "neutral", "negative"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"] + }, + + {"model_name": "slim-extract-tiny-onnx", "display_name": "slim-extract-tiny-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-extract-tiny-onnx", + "link": "https://huggingface.co/llmware/slim-extract-tiny-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/slim-extract-tiny", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [1066, 22198, 17821], + "marker_token_lookup": {1066: "positive", 22198: "negative", 17821: "neutral"}, + "function": ["extract"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"] + }, + + {"model_name": "slim-summary-tiny-onnx", "display_name": "slim-summary-tiny-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-summary-tiny-onnx", + "link": "https://huggingface.co/llmware/slim-summary-tiny-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "list", + "function_call": True, "primary_keys": ["key points (3)"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["summarize"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"] + }, + + {"model_name": "slim-sql-onnx", "display_name": "slim-sql-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-sql-onnx", + "fc_output_values": [], "link": "https://huggingface.co/llmware/slim-sql-onnx", + "custom_model_files": [], "custom_model_repo": "", "tokenizer": "llmware/slim-sql-1b-v0", + "tokenizer_local": "tokenizer_tl.json", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"] + }, + + {"model_name": "slim-emotions-onnx", "display_name": "slim-emotions-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-emotions-onnx", + "link": "https://huggingface.co/llmware/slim-emotions-tool", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["emotions"], + "fc_output_values": ["afraid", "anger", "angry", "annoyed", "anticipating", "anxious", "apprehensive", + "ashamed", "caring", "confident", "content", "devastated", "disappointed", "disgusted", + "embarrassed", "excited", "faithful", "fear", "furious", "grateful", "guilty", + "hopeful", "impressed", "jealous", "joy", "joyful", "lonely", "love", "nostalgic", + "prepared", "proud", "sad", "sadness", "sentimental", "surprise", "surprised", + "terrified", "trusting"], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], + "marker_token_lookup": {}, + "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"] + }, + + {"model_name": "slim-topics-onnx", "display_name": "slim-topics-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-topics-onnx", + "link": "https://huggingface.co/llmware/slim-topics-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, "primary_keys": ["topics"], "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, "function": ["classify"], + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"]}, + + {"model_name": "slim-ner-onnx", "display_name": "slim-ner-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, + "instruction_following": False, "prompt_wrapper": "human_bot", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/slim-ner-onnx", + "link": "https://huggingface.co/llmware/slim-ner-onnx", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "custom_model_files": [], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["people", "location", "organization", "misc"], + "fc_output_values": [], + "tokenizer": "llmware/slim-sentiment", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-intent-onnx", "display_name": "slim-intent-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-intent-onnx", + "link": "https://huggingface.co/llmware/slim-intent-onnx", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "custom_model_files":[], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["intent"], + "fc_output_Values": [], + "tokenizer": "llmware/slim-intent", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-tags-onnx", "display_name": "slim-tags-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-tags-onnx", + "link": "https://huggingface.co/llmware/slim-tags-onnx", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "custom_model_files":[], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["tags"], + "fc_output_Values": [], + "tokenizer": "llmware/slim-tags", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "slim-ratings-onnx", "display_name": "slim-ratings-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", "hf_repo": "llmware/slim-ratings-onnx", + "link": "https://huggingface.co/llmware/slim-ratings-onnx", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "custom_model_files":[], "custom_model_repo": "", + "output_type": "dict", "function_call": True, + "primary_keys": ["ratings"], + "fc_output_Values": [], + "tokenizer": "llmware/slim-ratings", + "tokenizer_local": "tokenizer_tl.json", + "marker_tokens": [], "marker_token_lookup": {}, + "function": ["classify"]}, + + {"model_name": "phi-3-onnx", + "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", + "display_name": "llmware/phi-3-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/phi-3-onnx"}, + + {"model_name": "llama-2-chat-onnx", + "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", + "display_name": "llmware/llama-2-chat-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/llama-2-chat-onnx", + "custom_model_files": [], "custom_model_repo": "", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/llama-2-chat-onnx"}, + + {"model_name": "llama-3.1-instruct-onnx", + "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", + "display_name": "llmware/llama-3.1-instruct-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "llama_3_chat", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/llama-3.1-instruct-onnx", + "custom_model_files": [], "custom_model_repo": "", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/llama-3.1-instruct-onnx"}, + + {"model_name": "dragon-mistral-0.3-onnx", + "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", + "display_name": "llmware/dragon-mistral-0.3-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "hf_repo": "llmware/dragon-mistral-0.3-onnx", + "custom_model_files": [], "custom_model_repo": "", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/dragon-mistral-0.3-onnx"}, + + {"model_name": "mistral-7b-instruct-v0.3-onnx", "display_name": "mistral-7b-instruct-v0.3-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/mistral-7b-instruct-v0.3-onnx", + "link": "https://huggingface.co/llmware/mistral-7b-instruct-v0.3-onnx", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "tiny-llama-chat-onnx", "display_name": "tiny-llama-chat-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 2048, "instruction_following": False, + "prompt_wrapper": "tiny_llama_chat", "temperature": 0.3, "trailing_space": "", + "gguf_repo": "llmware/tiny-llama-chat-onnx", + "link": "https://huggingface.co/llmware/tiny-llama-chat-onnx", + "tokenizer_local": "tokenizer_tl.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.2-1b-instruct-onnx", "display_name": "llama-3.2-1b-instruct-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "gguf_repo": "llmware/llama-3.2-1b-instruct-onnx", + "link": "https://huggingface.co/llmware/llama-3.2-1b-instruct-onnx", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.2-3b-instruct-onnx", "display_name": "llama-3.2-3b-instruct-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.3, "trailing_space": "", + "gguf_repo": "llmware/llama-3.2-3b-instruct-onnx", + "link": "https://huggingface.co/llmware/llama-3.2-3b-instruct-onnx", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "slim-boolean-phi-3-onnx", "display_name": "slim-boolean-phi-3-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-boolean-phi-3-onnx", + "link": "https://huggingface.co/llmware/slim-boolean-phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": [""], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["boolean"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [] + }, + + {"model_name": "slim-summary-phi-3-onnx", "display_name": "slim-summary-phi-3-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-summary-phi-3-onnx", + "link": "https://huggingface.co/llmware/slim-summary-phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["summarize"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + }, + + {"model_name": "slim-extract-phi-3-onnx", "display_name": "slim-extract-phi-3-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "human_bot", "temperature": 0.0, "sample_default": False, + "trailing_space": "", + "hf_repo": "llmware/slim-extract-phi-3-onnx", + "link": "https://huggingface.co/llmware/slim-extract-phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", "output_type": "dict", + "function_call": True, + "primary_keys": ["key points"], + "fc_output_values": [], + "tokenizer": "llmware/bling-phi-3", + "tokenizer_local": "tokenizer_phi3.json", + "function": ["extract"], + "snapshot": True, + "marker_tokens": [], + "marker_token_lookup": {}, + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": []}, + + {"model_name": "gemma-2b-it-onnx", "display_name": "gemma-2b-it-onnx", + "model_family": "ONNXGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "google_gemma_chat", "temperature": 0.3, "trailing_space": "", + "hf_repo": "llmware/gemma-2b-it-onnx", + "link": "https://huggingface.co/llmware/gemma-2b-it-onnx", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2-7B-instruct-gguf", "display_name": "qwen-2-7b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "qwen2-7b-instruct.gguf", "gguf_repo": "llmware/qwen2-7B-instruct-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qwen2-7b-instruct.gguf"], + "link": "https://huggingface.co/llmware/qwen2-7B-instruct-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 7.0}, + + {"model_name": "qwen2-1.5b-instruct-gguf", "display_name": "qwen-2-1.5b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "qwen-instruct-1-5b.gguf", + "gguf_repo": "llmware/qwen2-1.5b-instruct-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qwen-instruct-1-5b.gguf"], + "link": "https://huggingface.co/llmware/qwen2-1.5b-instruct-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 1.5}, + + {"model_name": "qwen2-0.5b-instruct-gguf", "display_name": "qwen-2-0.5b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 2048, "instruction_following": True, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "qwen2-0_5b-instruct-q4_k_m.gguf", "gguf_repo": "llmware/qwen-2-0.5b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["qwen2-0_5b-instruct-q4_k_m.gguf"], + "link": "https://huggingface.co/llmware/qwen-2-0.5b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 0.5}, + + {"model_name": "llama-3.2-1b-instruct-gguf", "display_name": "llama-3.2-1b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/llama-3.2-1b-gguf", + "gguf_file": "Llama-3.2-1B-Instruct-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/llama-3.2-1b-gguf", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 1.3, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "llama-3.2-3b-instruct-gguf", "display_name": "llama-3.2-3b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/llama3.2-3b-gguf", + "gguf_file": "Llama-3.2-3B-Instruct-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/llama3.2-3b-gguf", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 3.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen-2.5-7b-coder-gguf", "display_name": "qwen-coder-2.5-7b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen2.5-7b-coder-gguf", + "gguf_file": "Qwen2.5.1-Coder-7B-Instruct-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/qwen2.5-7b-coder-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 7.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen-2.5-14b-instruct-gguf", "display_name": "qwen-2.5-14b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen2.5-14b-instruct-gguf", + "gguf_file": "Qwen2.5-14B-Instruct-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/qwen2.5-14b-instruct-gguf", + "tokenizer_local": "tokenizer_qw.json", "parameters": 14.0, + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "mistral-7b-instruct-v0.3-gguf", "display_name": "mistral-0.3-7b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "temperature": 0.0, "trailing_space": "", + "gguf_file": "Mistral-7B-Instruct-v0.3-Q4_K_M.gguf", + "gguf_repo": "llmware/mistral-7b-instruct-v0.3-gguf", + "link": "https://huggingface.co/llmware/mistral-7b-instruct-v0.3-gguf", + "tokenizer_local": "tokenizer_mistral.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": "", "parameters": 7.3, + "tags": ["llmware-chat", "p7", "onnx", "green", "emerald"]}, + + {"model_name": "gemma-2-9b-instruct-gguf", "display_name": "gemma-2-9b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "google_gemma_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/gemma-2-9b-instruct-gguf", + "gguf_file": "gemma-2-9b-it-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/gemma-2-9b-instruct-gguf", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 9.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "gemma-2-27b-instruct-gguf", "display_name": "gemma-2-27b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "google_gemma_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/gemma-2-27b-instruct-gguf", + "gguf_file": "gemma-2-27b-it-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/gemma-2-27b-instruct-gguf", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 27.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "phi-4-gguf", "display_name": "phi-4-14b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": True, "prompt_wrapper": "phi_3", + "temperature": 0.0, "trailing_space": "", "gguf_file": "phi-4-Q4_K_M.gguf", + "gguf_repo": "llmware/phi-4-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "link": "https://huggingface.co/llmware/phi-4-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 14.0}, + + {"model_name": "phi-4-mini-gguf", "display_name": "phi-4-3b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": True, "prompt_wrapper": "phi_3", + "temperature": 0.0, "trailing_space": "", "gguf_file": "microsoft_Phi-4-mini-instruct-Q4_K_M.gguf", + "gguf_repo": "llmware/phi-4-mini-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["microsoft_Phi-4-mini-instruct-Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/phi-4-mini-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 3.8}, + + {"model_name": "deepseek-qwen-14b-gguf", "display_name": "deepseek-qwen-14b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "deepseek_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/deepseek-qwen-14b-gguf", + "gguf_file": "DeepSeek-R1-Distill-Qwen-14B-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/deepseek-qwen-14b-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 14.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "deepseek-qwen-7b-gguf", "display_name": "deepseek-qwen-7b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "deepseek_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/deepseek-qwen-7b-gguf", + "gguf_file": "DeepSeek-R1-Distill-Qwen-7B-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/deepseek-qwen-7b-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 7.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-32b-gguf", "display_name": "qwen2.5-32b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen2.5-32b-gguf", + "gguf_file": "Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/Qwen2.5-32B-Instruct-Q4_K_M.gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 32.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "gemma-4-26b-a4b-gguf", "display_name": "gemma-4-26b-a4b-moe", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 8192, "instruction_following": False, + "prompt_wrapper": "google_gemma_4_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/gemma-4-26B-a4b-gguf", + "gguf_file": "gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/gemma-4-26b-a4b-gguf/gemma-4-26B-A4B-it-UD-Q4_K_M.gguf", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 26.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3.6-35b-a3b-gguf", "display_name": "qwen3.6-35b-a3b-moe", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen3.6-35b-a3b-gguf", + "gguf_file": "Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/qwen3.6-35b-a3b-gguf/Qwen3.6-35B-A3B-UD-Q4_K_M.gguf", + "tokenizer_local": "tokenizer_gemma.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 35.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3.6-27b-gguf", "display_name": "qwen3.6-27b-gguf", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 8192, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen3.6-27b-gguf", + "gguf_file": "Qwen3.6-27B-Q4_K_M.gguf", + "link": "https://huggingface.co/llmware/qwen3.6-27b-gguf/Qwen3.6-27B-Q4_K_M.gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 27.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen3-vl-8b-gguf", "display_name": "qwen3-vl-vision-8b", + "model_family": "GGUFVisionGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 8192, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen-3-vl-8b-gguf", + "gguf_file": "Qwen3-VL-8B-Instruct-Q4_K_M.gguf", + "clip_file": "mmproj-F16.gguf", + "link": "https://huggingface.co/llmware/qwen-3-vl-8b-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 8.0, + "custom_model_files": [], "custom_model_repo": "", + "tags": ["llmware-chat", "p8", "gguf", "green", "emerald"]}, + + {"model_name": "qwen3-vl-4b-gguf", "display_name": "qwen3-vl-vision-4b", + "model_family": "GGUFVisionGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 8192, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen3-vl-4b-gguf", + "gguf_file": "Qwen3-VL-4B-Instruct-Q4_K_M.gguf", + "clip_file": "mmproj-F16.gguf", + "link": "https://huggingface.co/llmware/qwen3-vl-4b-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 4.0, + "custom_model_files": [], "custom_model_repo": "", + "tags": ["llmware-chat", "p4", "gguf", "green", "emerald"]}, + + {"model_name": "qwen3-vl-30b-gguf", "display_name": "qwen3-vl-vision-30b", + "model_family": "GGUFVisionGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 8192, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen-3-vl-30b-a3b-gguf", + "gguf_file": "Qwen3-VL-30B-A3B-Instruct-Q4_K_M.gguf", + "clip_file": "mmproj-F16.gguf", + "link": "https://huggingface.co/llmware/qwen-3-vl-30b-a3b-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 30.0, + "custom_model_files": [], "custom_model_repo": "", + "tags": ["llmware-chat", "p30", "gguf", "green", "emerald"]}, + + {"model_name": "qwen3-4b-instruct-gguf", "display_name": "qwen-3-4b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "Qwen3-4B-Q4_K_M.gguf", "gguf_repo": "llmware/qwen3-4b-instruct-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["Qwen3-4B-Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/qwen3-4b-instruct-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 4.0, + "tags": ["llmware-chat", "p4", "gguf", "green", "emerald"]}, + + {"model_name": "qwen3-8b-gguf", "display_name": "qwen-3-8b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "Qwen3-8B-Q4_K_M.gguf", "gguf_repo": "llmware/qwen3-8b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["Qwen3-8B-Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/qwen3-8b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 8.0, + "tags": ["llmware-chat", "p8", "gguf", "green", "emerald"]}, + + {"model_name": "qwen3-14b-gguf", "display_name": "qwen-3-14b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "Qwen3-14B-Q4_K_M.gguf", "gguf_repo": "llmware/qwen3-14b-gguf", + "validation_files": ["Qwen3-14B-Q4_K_M.gguf"], + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "link": "https://huggingface.co/llmware/qwen3-14b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 14.0, + "tags": ["llmware-chat", "p14", "gguf", "green", "emerald"]}, + + {"model_name": "qwen-3.5-4b-gguf", "display_name": "qwen-3.5-4b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "Qwen3.5-4B-Q4_K_M.gguf", "gguf_repo": "llmware/qwen-3.5-4b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["Qwen3.5-4B-Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/qwen-3.5-4b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 4.0, + "tags": ["llmware-chat", "p4", "gguf", "green", "emerald"]}, + + {"model_name": "qwen-3.5-9b-gguf", "display_name": "qwen-3.5-9b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "Qwen3.5-9B-Q4_K_M.gguf", "gguf_repo": "llmware/qwen-3.5-9b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["Qwen3.5-9B-Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/qwen-3.5-9b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 9.0, + "tags": ["llmware-chat", "p9", "gguf", "green", "emerald"]}, + + {"model_name": "qwen-3.5-27b-gguf", "display_name": "qwen-3.5-27b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "hf_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "Qwen3.5-27B-Q4_K_M.gguf", "gguf_repo": "llmware/qwen-3.5-27b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["Qwen3.5-27B-Q4_K_M.gguf"], + "link": "https://huggingface.co/llmware/qwen-3.5-27b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 27.0, + "tags": ["llmware-chat", "p27", "gguf", "green", "emerald"]}, + + {"model_name": "gpt-oss-20b-gguf", "display_name": "openai-oss-20b", + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", "model_location": "llmware_repo", + "context_window": 8192, "instruction_following": False, "prompt_wrapper": "oss_chat", + "temperature": 0.0, "trailing_space": "", + "gguf_file": "gpt-oss-20b-mxfp4.gguf", + "gguf_repo": "llmware/gpt-oss-20b-gguf", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "tokenizer_local": "tokenizer_phi3.json", # not used + "link": "https://huggingface.co/llmware/gpt-oss-20b-gguf", + "custom_model_files": [], "custom_model_repo": "", "parameters": 20.0, + "tags": ["llmware-chat", "p20", "gguf", "green", "emerald"]}, + + {"model_name": "llama-3.2-3b-onnx-qnn", "display_name": "llama-3.2-npu-3b", + "model_family": "ONNXQNNGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "llama_3_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/llama-3.2-3b-onnx-qnn", + "link": "https://huggingface.co/llmware/llama-3.2-3b-onnx-qnn", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "phi-3-mini-4k-instruct-onnx-qnn", + "display_name": "phi-3-mini-4k-instruct-onnx-qnn", + "model_family": "ONNXQNNGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "phi_3", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/phi-3-mini-4k-instruct-onnx-qnn", + "link": "https://huggingface.co/llmware/phi-3-mini-4k-instruct-onnx-qnn", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 3.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-1.5b-instruct-onnx-qnn", "display_name": "qwen2.5-1.5b-instruct-onnx-qnn", + "model_family": "ONNXQNNGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-1.5b-instruct-onnx-qnn", + "link": "https://huggingface.co/llmware/qwen2.5-1.5b-instruct-onnx-qnn", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 2.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "qwen2.5-7b-instruct-onnx-qnn", "display_name": "qwen2.5-7b-instruct-onnx-qnn", + "model_family": "ONNXQNNGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/qwen2.5-7b-instruct-onnx-qnn", + "link": "https://huggingface.co/llmware/qwen2.5-7b-instruct-onnx-qnn", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 7.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "deepseek-r1-distill-qwen-7b-onnx-qnn", + "display_name": "deepseek-r1-distill-qwen-7b-onnx-qnn", + "model_family": "ONNXQNNGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "deepseek_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/deepseek-r1-distill-qwen-7b-onnx-qnn", + "link": "https://huggingface.co/llmware/deepseek-r1-distill-qwen-7b-onnx-qnn", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 7.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "deepseek-r1-distill-qwen-14b-onnx-qnn", + "display_name": "deepseek-r1-distill-qwen-14b-onnx-qnn", + "model_family": "ONNXQNNGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "deepseek_chat", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/deepseek-r1-distill-qwen-14b-onnx-qnn", + "link": "https://huggingface.co/llmware/deepseek-r1-distill-qwen-14b-onnx-qnn", + "tokenizer_local": "tokenizer_ll3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 14.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "phi-3.5-mini-instruct-onnx-qnn", + "model_family": "ONNXQNNGenerativeModel", + "model_category": "generative_local", + "display_name": "phi-3.5-mini-instruct-onnx-qnn", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "onnx_file": "onnx-file", + "hf_repo": "llmware/phi-3.5-mini-instruct-onnx-qnn", + "custom_model_files": [], "custom_model_repo": "", + "snapshot": True, + "fetch": {"snapshot": True, "module": "llmware.models", + "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "link": "https://huggingface.co/llmware/phi-3.5-mini-instruct-onnx-qnn"}, + + {"model_name": "jina-reranker-tiny-onnx", + "model_family": "ONNXEmbeddingModel", + "model_category": "embedding", + "display_name": "jina-reranker-tiny-onnx", + "model_location": "llmware_repo", "embedding_dims": 384, + "context_window": 8192, "use_case": "ranker", + "hf_repo": "llmware/jina-reranker-tiny-onnx", + "custom_model_files": [], "custom_model_repo": "", + "link": "https://huggingface.co/llmware/jina-reranker-tiny-onnx"}, + + {"model_name": "jina-reranker-turbo-onnx", + "model_family": "ONNXEmbeddingModel", + "model_category": "embedding", + "display_name": "jina-reranker-turbo-onnx", + "model_location": "llmware_repo", "embedding_dims": 384, + "context_window": 8192, "use_case": "ranker", + "hf_repo": "llmware/jina-reranker-turbo-onnx", + "custom_model_files": [], "custom_model_repo": "", + "link": "https://huggingface.co/llmware/jina-reranker-turbo-onnx"}, + + {"model_name": "protectai-prompt-injection-onnx", + "display_name": "protectai-prompt-injection-onnx", + "model_family": "ONNXEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "hf_repo": "llmware/protectai-prompt-injection-onnx", "use_case": "classifier", + "custom_model_repo": "", "pytorch_model_repo": "protectai/deberta-v3-base-prompt-injection"}, + + {"model_name": "valurank-bias-onnx", "display_name": "valurank-bias-onnx", + "model_family": "ONNXEmbeddingModel", "model_category": "embedding", + "model_location": "llmware_repo", "use_case": "classifier", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "pytorch_model_repo": "valurank/distilroberta-bias", + "hf_repo": "llmware/valurank-distilroberta-bias-onnx"}, + + {"model_name": "unitary-toxic-roberta-onnx", "display_name": "unitary-toxic-roberta-onnx", + "model_family": "ONNXEmbeddingModel", "model_category": "embedding", + "model_location": "llmware_repo", "use_case": "classifier", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/unitary-unbiased-toxic-roberta-onnx"}, + + {"model_name": "phi-3-vision-onnx", "display_name": "phi-3-vision-3b", + "model_family": "ONNXVisionGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "phi_3_vision", "temperature": 0.0, "trailing_space": "", + "hf_repo": "llmware/phi-3-vision-onnx", + "link": "https://huggingface.co/llmware/phi-3-vision-onnx", + "tokenizer_local": "tokenizer_phi3.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["phi-3-v-128k-instruct-text.onnx.data", + "phi-3-v-128k-instruct-vision.onnx.data", + "phi-3-v-128k-instruct-embedding.onnx.data"], + "custom_model_files": [], "custom_model_repo": "", "parameters": 3.8}, + + {"model_name": "jina-reranker-v1-tiny-en-ov", "display_name": "jina-reranker-v1-tiny-en-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 384, "context_window": 8192, "use_case": "ranker", + "link": "https://www.huggingface.com/llmware/jina-reranker-v1-tiny-en-ov", + "custom_model_repo": "", "hf_repo": "llmware/jina-reranker-v1-tiny-en-ov"}, + + {"model_name": "jina-reranker-v1-turbo-en-ov", "display_name": "jina-reranker-v1-turbo-en-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 384, "context_window": 8192, "use_case": "ranker", + "link": "https://www.huggingface.com/llmware/jina-reranker-v1-turbo-en-ov", + "custom_model_repo": "", "hf_repo": "llmware/jina-reranker-v1-turbo-en-ov"}, + + {"model_name": "industry-bert-contracts-ov", "display_name": "industry-bert-contracts-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/industry-bert-contracts-ov"}, + + {"model_name": "industry-bert-insurance-ov", "display_name": "industry-bert-insurance-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/industry-bert-insurance-ov"}, + + {"model_name": "industry-bert-asset-management-ov", "display_name": "industry-bert-asset-management-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/industry-bert-asset-management-ov"}, + + {"model_name": "industry-bert-sec-ov", "display_name": "industry-bert-sec-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/industry-bert-sec-ov"}, + + {"model_name": "industry-bert-loans-ov", "display_name": "industry-bert-loans-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/industry-bert-loans-ov"}, + + {"model_name": "all-mini-lm-l6-v2-ov", "display_name": "all-mini-lm-l6-v2-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 384, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/all-mini-lm-l6-v2-ov"}, + + {"model_name": "all-mpnet-base-v2-ov", "display_name": "all-mpnet-base-v2-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 514, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/all-mpnet-base-v2-ov"}, + + {"model_name": "paraphrase-multilingual-MiniLM-L12-v2-ov", + "display_name": "paraphrase-multilingual-MiniLM-L12-v2-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 384, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/paraphrase-multilingual-MiniLM-L12-v2-ov"}, + + {"model_name": "gte-small-ov", "display_name": "gte-small-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 384, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/gte-small-ov"}, + + {"model_name": "gte-base-ov", "display_name": "gte-base-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/gte-base-ov"}, + + {"model_name": "gte-large-ov", + "display_name": "gte-large-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 1024, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/gte-large-ov"}, + + {"model_name": "bge-small-en-v1.5-ov", + "display_name": "bge-small-en-v1.5-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 384, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/bge-small-en-v1.5-ov"}, + + {"model_name": "bge-base-en-v1.5-ov", + "display_name": "bge-base-en-v1.5-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/bge-base-en-v1.5-ov"}, + + {"model_name": "bge-large-en-v1.5-ov", + "display_name": "bge-large-en-v1.5-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 1024, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/bge-large-en-v1.5-ov"}, + + {"model_name": "protectai-prompt-injection-ov", "display_name": "protectai-prompt-injection-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", "model_location": "llmware_repo", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "hf_repo": "llmware/protectai-prompt-injection-ov", "use_case": "classifier", + "custom_model_repo": "", "pytorch_model_repo": "protectai/deberta-v3-base-prompt-injection"}, + + {"model_name": "xlm-roberta-language-detector-ov", + "display_name": "xlm-roberta-language-detector-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", + "model_location": "llmware_repo", "use_case": "classifier", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "pytorch_repo": "papluca/xlm-roberta-base-language-detection", + "hf_repo": "llmware/xlm-roberta-language-detector-ov"}, + + {"model_name": "valurank-bias-ov", "display_name": "valurank-bias-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", + "model_location": "llmware_repo", "use_case": "classifier", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "pytorch_model_repo": "valurank/distilroberta-bias", + "hf_repo": "llmware/valurank-bias-ov"}, + + {"model_name": "unitary-toxic-roberta-ov", "display_name": "unitary-toxic-roberta-ov", + "model_family": "OVEmbeddingModel", "model_category": "embedding", + "model_location": "llmware_repo", "use_case": "classifier", + "embedding_dims": 768, "context_window": 512, "link": "https://none", + "custom_model_repo": "", "hf_repo": "llmware/unitary-toxic-roberta-ov"}, + + {"model_name": "qwen2.5-vl-3b-ov", "model_family": "OVVisionGenerativeModel", + "model_category": "generative_local", "display_name": "qwen-2.5-vl-3b", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "tokenizer_local": "tokenizer_qw.json", + "hf_repo": "llmware/qwen2.5-vl-3b-ov", "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_language_model.bin", + "openvino_text_embeddings_model.bin", + "openvino_vision_embeddings_merger_model.bin"], + "parameters": 3.0, + "link": "https://huggingface.co/llmware/qwen2.5-vl-3b-ov" + }, + + {"model_name": "phi-3.5-vision-ov", "model_family": "OVVisionGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3.5-vision-3b", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3.5-vision-ov", "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", + "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "parameters": 3.0, + "link": "https://huggingface.co/llmware/phi-3.5-vision-ov" + }, + + {"model_name": "phi-4-mm-ov", "model_family": "OVVisionGenerativeModel", + "model_category": "generative_local", "display_name": "phi-4-mm-ov", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "tokenizer_local": "tokenizer_phi4.json", + "hf_repo": "llmware/phi-4-mm-ov", "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", + "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "parameters": 6.0, + "link": "https://huggingface.co/llmware/phi-4-mm-ov" + }, + + {"model_name": "gemma-3-4b-ov", "model_family": "OVVisionGenerativeModel", + "model_category": "generative_local", "display_name": "gemma-3-4b", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "", "tokenizer_local": "tokenizer_gemma.json", + "hf_repo": "llmware/gemma-3-4b-ov", "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", + "method": "pull_snapshot_from_hf"}, + "validation_files": [], + "parameters": 4.0, + "link": "https://huggingface.co/llmware/gemma-3-4b-ov" + }, + + {"model_name": "qwen2.5-vl-3b-instruct-gguf", "display_name": "qwen2.5-vl-vision-model", + "model_family": "GGUFVisionGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/qwen2.5-vl-3b-instruct-gguf", + "gguf_file": "Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf", + "clip_file": "mmproj-F16.gguf", + "link": "https://huggingface.co/llmware/qwen2.5-vl-3b-instruct-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 3.0, + "custom_model_files": [], "custom_model_repo": ""}, + + {"model_name": "minicpm-2.6-gguf", "display_name": "minicpm-vision-model", + "model_family": "GGUFVisionGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": 4096, "instruction_following": False, + "prompt_wrapper": "hf_chat", "temperature": 0.0, "trailing_space": "", + "gguf_repo": "llmware/minicpm-2.6-gguf", + "gguf_file": "MiniCPM-V-2_6-Q4_K_M.gguf", + "clip_file": "mmproj-model-f16-2.gguf", + "link": "https://huggingface.co/llmware/minicpm-2.6-gguf", + "tokenizer_local": "tokenizer_qw.json", + "fetch": {"module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": [], "parameters": 7.0, + "custom_model_files": [], "custom_model_repo": ""} + +] + +""" Fine-tuning Prompt Wrappers - virtually all instruct fine-tuned models will have a special 'prompt wrapper' +that is an artifact from fine-tuning and needs to be applied consistently to lead to the expected model behavior. +There are a number of common formats captured in the default catalog, but can be extended through ModelCatalog. +When constructing the prompt, this wrapper will be applied automatically. """ + +global_model_finetuning_prompt_wrappers_lookup = { + + # each wrapper can consist of up to 5 elements to represent common segments of the prompt + # 1. optional - "system_start" and "system_stop" + # 2. required - "main_start" and "main_stop" + # 3. required - "start_llm_response" + + "human_bot": {"main_start": ": ", "main_stop": "\n", "start_llm_response": ":"}, + + # update for mistral models + "": {"system_start": "[SYSTEM_PROMPT]", + "system_stop": "[/SYSTEM_PROMPT]", + "main_start": "", + "main_stop": "", + "start_llm_response": ""}, + + "hf_chat": {"system_start": "<|im_start|>system\n", "system_stop": "<|im_end|>\n", + "main_start": "<|im_start|>user", "main_stop": "<|im_end|>\n", + "start_llm_response": "<|im_start|>assistant"}, + + "oss_chat": {"system_start": """<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI. + \nKnowledge cutoff: 2024-06\nCurrent date: 2025-08-05\n\nReasoning: medium\n\nChannel: final\n\n + # Valid channels: analysis, commentary, final""", + "system_stop": "<|end|>", + "main_start": "<|start|>user<|message|>", "main_stop": "<|end|>", + # <|channel|>analysis<|message|><|end|><|start|>assistant + "start_llm_response": "<|channel|>analysis<|message|><|end|><|start|>assistant"}, + + "lfm2_chat": {"system_start": "<|startoftext|><|im_start|>system", "system_stop": "<|im_end|>", + "main_start": "<|im_start|>user", "main_stop": "|im_end|>", + "start_llm_response": "<|im_start|>assistant"}, + + "olmo_chat": {"system_start": "<|endoftext|><|system|>\n", "system_stop": "", + "main_start": "<|user|>\n", "main_stop": "", + "start_llm_response": "<|assistant|>\n"}, + + "granite_chat": {"system_start": "<|start_of_role|>system<|end_of_role|>", + "system_stop": "<|end_of_text|>\n", + "main_start": "<|start_of_role|>user<|end_of_role|>", + "main_stop": "<|end_of_text|>\n", + "start_llm_response": "<|start_of_role|>assistant<|end_of_role|>"}, + + "open_chat": {"main_start": "GPT4 User: ", "main_stop": "<|endofturn|>", + "start_llm_response": "GPT4 Assistant:"}, + + "alpaca": {"main_start": "### Instruction: ", "main_stop": "\n", + "start_llm_response": "### Response: "}, + + "chat_ml": {"system_start": "<|im_start|>system", "system_stop":"<|im_end|>\n", + "main_start":"<|im_start|>user", "main_stop":"<|im_end|>\n", + "start_llm_response":"<|im_start|>assistant"}, + + "phi_3": {"system_start": "<|system|>\n", "system_stop": "<|end|>\n", + "main_start": "<|user|>\n", "main_stop": "<|end|>\n", "start_llm_response": "<|assistant|>"}, + + # intended for embedding one image only currently + "phi_3_vision": {"system_start": "", "system_stop": "", + "main_start": "<|user|>\n<|image_1|>\n", + "main_stop": "<|end|>\n", + "start_llm_response": "<|assistant|>\n"}, + + "phi_4": {"system_start": "<|im_start|>system<|im_sep|>\n", + "system_stop": "<|im_end|>\n", + "main_start": "<|im_start|>user<|im_sep|>\n", + "main_stop": "<|im_end|>\n", + "start_llm_response": "<|im_start|>assistant<|im_sep|>"}, + + "phi_4_mini": {"system_start": "<|system|>", "system_stop": "<|end|>", + "main_start": "<|user|>", "main_stop": "<|end|>", + "start_llm_response": "<|assistant|>"}, + + "llama_3_chat": {"system_start": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n", + "system_stop": "<|eot_id|>", + "main_start": "<|start_header_id|>user>|end_header_id|>\n", + "main_stop": "<|eot_id|>", + "start_llm_response": "<|start_header_id|>assistant<|end_header_id|>\n"}, + + "tiny_llama_chat": {"system_start": "<|system|>", "system_stop": "", + "main_start": "<|user|>", "main_stop": "", + "start_llm_response": "<|assistant|>"}, + + "stablelm_zephyr_chat": {"system_start": "", "system_stop": "", + "main_start": "<|user|>", "main_stop": "<|endoftext|>\n", + "start_llm_response": "<|assistant|>"}, + + "google_gemma_chat": {"system_start": "", "system_stop": "", + "main_start": "user\n", + "main_stop": "\n", + "start_llm_response": "model"}, + + "google_gemma_4_chat": {"system_start": "<|turn>system\n", "system_stop": "\n", + "main_start": "<|turn>user\n", "main_stop": "\n", + "start_llm_response": "<|turn>model\n"}, + + "vicuna_chat": {"system_start": "", "system_stop": "", + "main_start": "USER: ", "main_stop": "", + "start_llm_response": " ASSISTANT:"}, + + "deepseek_chat": {"system_start": "<|begin_of_sentence|>", "system_stop": "", + "main_start": "<|User|>", "main_stop": "", + "start_llm_response": "<|Assistant|>"} + +} + +""" Tokenizer EOS/BOS lookup master table """ + +global_tokenizer_bos_eos_lookup = { + + "tokenizer_phi3.json": {"bos_id": 1, "bos_token": "", + "eos_id": [32000, 32001, 32007], "eos_token": "<|endoftext|>"}, + + # removing 100265 = <|im_end|> + "tokenizer_phi4.json": {"bos_id": 100257, "bos_token": "<|endoftext|>", + "eos_id": [100257, 100265], "eos_token": "<|endoftext|>"}, + + "tokenizer_phi4_mini.json": {"bos_id": 199999, "bos_token": "<|endoftext|>", + "eos_id": [199999, 200020], "eos_token": "<|endoftext|>"}, + + "tokenizer_stablelm.json": {"bos_id": 0, "bos_token": "<|endoftext|>", + "eos_id": [0], "eos_token": "<|endoftext|>"}, + + "tokenizer_stablelm_1_6.json": {"bos_id": 100257, "bos_token": "<|endoftext|>", + "eos_id": [100257], "eos_token": "<|endoftext|>"}, + + "tokenizer_tl.json": {"bos_id": 1, "bos_token": "", + "eos_id": [2, 32000], "eos_token": ""}, + + "tokenizer_ll2.json": {"bos_id": 1, "bos_token": "", + "eos_id": [2], "eos_token": ""}, + + "tokenizer_gemma.json": {"bos_id": 2, "bos_token": "", + "eos_id": [1], "eos_token": ""}, + + "tokenizer_ll3.json": {"bos_id": 128000, "bos_token": "<|begin_of_text|>", + "eos_id": [128001, 128008, 128009, 128256], "eos_token": "<|eot_id|>" + }, + + "tokenizer_qw.json": {"bos_id": 151643, "bos_token": "<|endoftext|>", + "eos_id": [151643, 151645], + "eos_token": ["<|im_end|>"]}, + + "tokenizer_phi2.json": {"bos_id": 50256, "bos_token": "<|endoftext|>", + "eos_id": [50256], "eos_token": "<|endoftext|>"}, + + "tokenizer_granite.json": {"bos_id": 100257, "bos_token": "<|end_of_text|>", + "eos_id": [100257], "eos_token": "<|end_of_text|>"}, + + # 01-ai yi tokenizer + "tokenizer_yi.json": {"bos_id": 1, "bos_token": "<|startoftext|>", + "eos_id": [2, 7], "eos_token": "<|endoftext|>"}, + + # Mistral tokenizer + "tokenizer_mistral.json": {"bos_id": 1, "bos_token": "", + "eos_id": [2], "eos_token": ""}, + + "tokenizer_mistral_chat.json": {"bos_id": 1, "bos_token": "", + "eos_id": [2, 32000, 32768], "eos_token": ["", "<|im_end|>"]}, + +} + + +""" Global default prompt catalog consists of a set of prebuilt useful prompt instructions across a wide range +of models. Unlike prompt_wrappers, which tend to be an attribute of the model, the prompt catalog can be invoked +on a 'prompt-by-prompt' basis to drive different behavior from a model. Note: not all models will support + very complex open-ended instructions or respond in a consistent manner. """ + +global_default_prompt_catalog = [ + + {"prompt_name": "just_the_facts", + "prompt_description": "Closed Context - read passage, answer question, stick to the facts.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following text: ", + "blurb2": " Please answer the question: ", + "instruction": "In providing the answer, please only use facts contained in the text.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {}}, + + {"prompt_name": "answer_or_not_found", + "prompt_description": "Closed Context - read passage, answer question, provide 'Not Found' if no answer in text.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following text: ", + "blurb2": " Please answer the question: ", + "instruction": "Please only use facts in the text. If the text does not provide the answer, then please " + "respond with: {{not_found_response}}", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {"not_found_response": "'Not Found.'"}}, + + {"prompt_name": "number_or_none", + "prompt_description": "Closed Context - read passage, answer question, provide 'Not Found' if no answer in text.", + "run_order": ["blurb1", "$context", "blurb2", "$query","instruction"], + "blurb1" : "Please read the following text: ", + "blurb2" : " Please answer the question: ", + "instruction": "Please provide a specific number as an answer from the text. " + "If the text does not provide a specific numerical answer, then please respond " + "with: {{not_found_response}}", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {"not_found_response": "'Not Found.'"}}, + + {"prompt_name": "summarize_with_bullets", + "prompt_description": "Basic summarization with open ended number of bullet points.", + "run_order": ["blurb1", "$context", "instruction"], + "blurb1": "Please read the following text: ", + "instruction": "Please summarize with bulletpoints.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {}}, + + {"prompt_name": "summarize_with_numbered_bullets", + "prompt_description": "Summarization with specified number of bullet points.", + "run_order": ["blurb1", "$context", "instruction"], + "blurb1": "Please read the following text: ", + "instruction": "Please summarize the text with approximately {{number_of_bulletpoints}} numbered bulletpoints.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {"number_of_bulletpoints": 5}}, + + {"prompt_name": "xsummary", + "prompt_description": "Xtreme summarization with specified number of words.", + "run_order": ["blurb1", "$context", "instruction"], + "blurb1": "Please read the following text: ", + "instruction": "Please summarize the text in no more than {{number_of_words}} words.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {"number_of_words": 25}}, + + {"prompt_name": "completion", + "prompt_description": "Open context text generation to complete starting point provided in prompt.", + "run_order": ["blurb1", "$query", "instruction"], + "blurb1": "Here is the starting point of a longer text: ", + "instruction": "Please complete this text in the style provided in the text.", + "system_message": "You are a helpful assistant who is a good creative writer.", + "user_vars": {}}, + + {"prompt_name": "dialog_summary", + "prompt_description": "General summarization of a conversation text with specified number of bullet points.", + "run_order": ["blurb1", "$context", "instruction"], + "blurb1": "Please read the following discussion between two parties: ", + "instruction": "Please summarize the key points from the conversation using less " + "than {{number_of_bulletpoints}} bulletpoints.", + "system_message": "You are a helpful assistant.", + "user_vars": {"number_of_bulletpoints": 10}}, + + {"prompt_name": "not_found_classifier", + "prompt_description": "Not Found Response classifier - used to ask a model to classify a particular response " + "as 'not found' - very useful in RAG applications.", + "run_order": ["blurb1", "blurb2", "$context", "instruction"], + "blurb1": "Here are several examples of a 'not found' response: " + "Not Found \n" + "The text does not provide an answer. \n" + "The answer is not clear. \n" + "Sorry, I could not find a definitive answer. \n" + "The answer is not provided in the information given. \n" + "The text does not specify the answer to this question. \n", + "blurb2": "Here is a new example: ", + "instruction": "Please respond 'Yes' or 'No' if this new example is a 'Not Found' response.", + "system_message": "You are a helpful assistant.", + "user_vars": {}}, + + {"prompt_name": "top_level_select", + "prompt_description": "Select the best answer among choices provided.", + "run_order": ["blurb1", "$query", "blurb2","$context", "instruction"], + "blurb1": "We are trying to answer the following question: ", + "blurb2": "Which of the following selections best answers the question?", + "instruction": "Please respond with the best answer among these selections. " + "If more than one answer is useful, please summarize with bulletpoints.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {}}, + + {"prompt_name": "answer_question_in_role", + "prompt_description": "Answer a question with a specific role or point of view.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following text: ", + "blurb2": "Please answer the following question: ", + "instruction": "In providing an answer to the question, please assume the perspective of a {{role}} and " + "write in that style.", + "system_message": "You are a helpful assistant.", + "user_vars": {"role": "business analyst"}}, + + {"prompt_name": "editor_in_role", + "prompt_description": "Edit a passage with a specific role or point of view.", + "run_order": ["blurb1", "$context", "instruction"], + "blurb1": "Please read the following text: ", + "instruction": "Our task is to edit and improve the language of the text from the perspective of a business analyst.", + "system_message": "You are a helpful editor and writer who reads text and improves the writing.", + "user_vars": {"role": "business analyst"}}, + + {"prompt_name": "yes_no", + "prompt_description": "Answer a question with 'Yes' or 'No'.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following text: ", + "blurb2": "Based on these materials, please answer the question: ", + "instruction": "Please answer this question with 'Yes' or 'No'. If the text does not provide an answer," + "then please respond with 'Not Found.'", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words.", + "user_vars": {}}, + + {"prompt_name": "multiple_choice", + "prompt_description": "Answer a question using a set of pre-defined choices provided.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following text: ", + "blurb2": "Based on these materials, please answer the question: ", + "instruction": "Please select from the choices provided. If the text does not provide an answer," + "then please respond with 'Not Found.'", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words."}, + + {"prompt_name": "default_with_context", + "prompt_description": "Default simple prompt when a question and context are passed.", + "run_order": ["blurb1", "$context", "blurb2", "$query"], + "blurb1": "Please read the following text: ", + "blurb2": "Based on this text, please answer the question: ", + "instruction": "", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words."}, + + {"prompt_name": "default_no_context", + "prompt_description": "Default simple prompt when only a question is passed.", + "run_order": ["blurb1","$query"], + "blurb1": "Please discuss the following: ", + # "blurb2": "Based on this text, please answer the question: ", + "instruction": "", + "system_message": "You are a helpful assistant who likes to answer questions."}, + + {"prompt_name": "summarize_with_bullets_w_query", + "prompt_description": "Summarization of a text with a specific question being posed.", + "run_order": ["blurb1", "$context", "blurb2","$query","instruction"], + "blurb1": "Please read the following text: ", + "blurb2": "Please read the following question: ", + "instruction": "Please summarize with bulletpoints an analysis of the question.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words."}, + + {"prompt_name": "summarize_with_references_w_query", + "prompt_description": "Summarization with text with guidance to provide reference to specific " + "information in the text passage.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following text: ", + "blurb2": "Please read the following question: ", + "instruction": "Please provide an analysis of the question using information and specific clauses " + "in the text.", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words."}, + + {"prompt_name": "write_poem", + "prompt_description": "Write a poem prompt - note: results may vary greatly by model.", + "run_order": ["instruction", "$query"], + "instruction": "Please write a poem using the following prompt: ", + "system_message": "You are a helpful assistant who is a creative writer and can rhyme words easily."}, + + {"prompt_name": "ten_words", + "prompt_description": "Xtreme summarization to answer question from a text in 10 words of less.", + "run_order": ["instruction", "$query", "$context"], + "blurb1": "Please read the following text: ", + "blurb2": "Please read the following question: ", + "instruction": "In no more than ten words, please give concise answer to the following question, using the " + "text as evidence to support", + "system_message": "You are a helpful assistant who speaks with facts and no wasted words."}, + + {"prompt_name": "explain_child", + "prompt_description": "Standard simplified answer prompt - note: results may vary greatly by model.", + "run_order": ["instruction", "$query", "$context"], + "instruction": "Please explain to a child the following question using the provided text: ", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "make_joke", + "prompt_description": "Standard joke prompt - note: results may vary greatly by model.", + "run_order": ["instruction", "$query"], + "instruction": "Please be funny and tell a joke on the subject of: ", + "system_message": "You are a helpful assistant with a good sense of humor."}, + + {"prompt_name": "tell_story", + "prompt_description": "Standard tell a story prompt - note: results may vary greatly by model.", + "run_order": ["instruction", "$query"], + "instruction": "Please write the start of a story on the topic of: ", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "write_headline", + "prompt_description": "Generate a headline from a question and context.", + "run_order": ["instruction", "$query", "$context"], + "instruction": "Please write the headline only in a few words in capitalization to answer the question below, " + "using the materials provided. ", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "facts_only", + "prompt_description": "Basic 'facts only' Q&A prompt.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please use the following materials- ", + "blurb2": "Please answer the following question - ", + "instruction": "In answering the question, please only use information contained in the provided materials.", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "top_bulletpoints", + "prompt_description": "Summarization with question and answer in 5 bullet points.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the text below - ", + "blurb2": "Please read the following question - ", + "instruction": "Please answer the question using the text, and write no more than 5 bulletpoints.", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "report_title", + "prompt_description": "Generate title of report given context passage.", + "run_order": ["instruction", "$context"], + "instruction": "Please write the title to a report with the following information: ", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "marketing_slogan", + "prompt_description": "Generate marketing style slogan given context passage.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following materials- ", + "blurb2": "Please answer the following question - ", + "instruction": "Please write a marketing slogan for the following offering using the following information as " + "background source materials.", + "system_message": "You are a helpful assistant."}, + + {"prompt_name": "top_level_summary", + "prompt_description": "Summarization prompt intended for 'second-level' summaries of materials.", + "run_order": ["blurb1", "$context", "blurb2", "$query", "instruction"], + "blurb1": "Please read the following materials- ", + "blurb2": "Please answer the following question - ", + "instruction": "In answering the question, please write no more than five bulletpoints, and reference the most " + "important facts in the source materials.", + "system_message": "You are a helpful assistant."}, + +] + + +model_benchmark_data = [ + + {"model_name": "bling-phi-3-gguf", + "base_model": "microsoft/Phi-3-mini-4k-instruct", + "parameters": 3.8, + "accuracy_score": 100, + "not_found": 0.95, + "yes_no": 0.975, + "math_logic": 0.80, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "bling-phi-3.5-gguf", + "base_model": "microsoft/Phi-3.5-mini-instruct", + "parameters": 3.8, + "accuracy_score": 100, + "not_found": 0.85, + "yes_no": 0.95, + "math_logic": 0.90, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "dragon-yi-6b-v0", + "base_model": "01-ai/yi-6b-v1", + "parameters": 6.0, + "accuracy_score": 99.5, + "not_found": 0.90, + "yes_no": 0.875, + "math_logic": 0.775, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "dragon-mistral-0.3-gguf", + "base_model": "mistralai/Mistral-7B-v0.3", + "parameters": 7.0, + "accuracy_score": 99.5, + "not_found": 0.90, + "yes_no": 0.825, + "math_logic": 0.675, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "dragon-qwen2-7b-gguf", + "base_model": "qwen/Qwen2-7b", + "parameters": 7.0, + "accuracy_score": 99, + "not_found": 0.85, + "yes_no": 1.0, + "math_logic": 0.925, + "complex_qa": 5, + "summarization": 4}, + + {"model_name": "dragon-yi-9b-gguf", + "base_model": "01-ai/yi-v1.5-9b", + "parameters": 8.8, + "accuracy_score": 98, + "not_found": 0.90, + "yes_no": 0.925, + "math_logic": 0.95, + "complex_qa": 5, + "summarization": 4}, + + {"model_name": "dragon-deci-7b", + "base_model": "Deci/Deci-7B", + "parameters": 7.0, + "accuracy_score": 97.5, + "not_found": 0.95, + "yes_no": 0.925, + "math_logic": 0.9125, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "dragon-llama-7b-v0", + "base_model": "meta-llama/llama-2-base", + "parameters": 7.0, + "accuracy_score": 97.25, + "not_found": 0.925, + "yes_no": 0.95, + "math_logic": 0.6375, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "dragon-mistral-7b-v0", + "base_model": "mistralai/mistral-7b-base-0.1", + "parameters": 7.0, + "accuracy_score": 96.5, + "not_found": 0.925, + "yes_no": 0.9750, + "math_logic": 0.8125, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "dragon-red-pajama-7b-v0", + "base_model": "togethercomputer/RedPajama-INCITE-7B-Base", + "parameters": 7.0, + "accuracy_score": 96, + "not_found": 0.55, + "yes_no": 0.8125, + "math_logic": 0.5250, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "dragon-deci-6b", + "base_model": "Deci/Deci-6B", + "parameters": 6.0, + "accuracy_score": 94.25, + "not_found": 0.775, + "yes_no": 0.9625, + "math_logic": 0.6875, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "dragon-llama-8b-3.1-gguf", + "base_model": "meta-llama/meta-llama-8b-3.1-base", + "parameters": 8.0, + "accuracy_score": 94, + "not_found": 0.70, + "yes_no": 0.90, + "math_logic": 0.7250, + "complex_qa": 4, + "summarization": 4}, + + {"model_name": "dragon-stablelm-7b-v0", + "base_model": "StableLM-7b-v2", + "parameters": 7.0, + "accuracy_score": 94, + "not_found": 0.85, + "yes_no": 0.8875, + "math_logic": 0.6250, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "dragon-falcon-7b-v0", + "base_model": "tiiuae/falcon-7b", + "parameters": 7.0, + "accuracy_score": 94, + "not_found": 0.75, + "yes_no": 0.8125, + "math_logic": 0.6675, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-stablelm-3b", + "base_model": "stabilityai/stablelm-3b-4e1t", + "parameters": 2.8, + "accuracy_score": 94, + "not_found": 0.675, + "yes_no": 0.78, + "math_logic": 0.29, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-qwen-mini-tool", + "base_model": "Qwen/Qwen2-1.5b", + "parameters": 1.5, + "accuracy_score": 93.5, + "not_found": 0.75, + "yes_no": 0.875, + "math_logic": 0.70, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-phi-2", + "base_model": "microsoft/phi-2", + "parameters": 2.8, + "accuracy_score": 93, + "not_found": 0.95, + "yes_no": 0.850, + "math_logic": 0.8250, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-red-pajamas-3b", + "base_model": "togethercomputer/RedPajama-INCITE-Instruct-3B-v1", + "parameters": 2.8, + "accuracy_score": 92, + "not_found": 0.45, + "yes_no": 0.75, + "math_logic": 0.20, + "complex_qa": 2, + "summarization": 3}, + + {"model_name": "bling-sheared-llama-2.7b", + "base_model": "princeton-nlp/Sheared-LLaMA-2.7B", + "parameters": 2.7, + "accuracy_score": 90.25, + "not_found": 0.60, + "yes_no": 0.80, + "math_logic": 0.50, + "complex_qa": 2, + "summarization": 3}, + + {"model_name": "bling-falcon-1b", + "base_model": "tiiuae/falcon-1b", + "parameters": 1.3, + "accuracy_score": 89, + "not_found": 0.575, + "yes_no": 0.58, + "math_logic": 0.25, + "complex_qa": 1, + "summarization": 3}, + + {"model_name": "bling-phi-1.5", + "base_model": "microsoft/phi-1.5", + "parameters": 1.5, + "accuracy_score": 87.75, + "not_found": 0.475, + "yes_no": 0.80, + "math_logic": 0.5375, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-tiny-llama-v0", + "base_model": "tinyllama/tinyllama-3T-1.1-v0[confirm]", + "parameters": 1.1, + "accuracy_score": 86.5, + "not_found": 0.85, + "yes_no": 0.825, + "math_logic": 0.3750, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-sheared-llama-1.3b", + "base_model": "princeton-nlp/Sheared-LLaMA-1.3B", + "parameters": 1.3, + "accuracy_score": 84.5, + "not_found": 0.20, + "yes_no": 0.6625, + "math_logic": 0.0940, + "complex_qa": 1, + "summarization": 3}, + + {"model_name": "bling-qwen-nano-tool", + "base_model": "Qwen/Qwen2-0.5b", + "parameters": 0.5, + "accuracy_score": 81, + "not_found": 0.65, + "yes_no": 0.6250, + "math_logic": 0.4250, + "complex_qa": 3, + "summarization": 3}, + + {"model_name": "bling-1b-0.1", + "base_model": "EleutherAI/pythia-1b", + "parameters": 1.0, + "accuracy_score": 73.25, + "not_found": 0.1750, + "yes_no": 0.29, + "math_logic": 0.0, + "complex_qa": 1, + "summarization": 1}, + + {"model_name": "bling-1.4b-0.1", + "base_model": "EleutherAI/pythia-1.4b", + "parameters": 1.4, + "accuracy_score": 82.25, + "not_found": 0.40, + "yes_no": 0.6125, + "math_logic": 0.0875, + "complex_qa": 1, + "summarization": 2} +] + diff --git a/llmware/models.py b/llmware/models.py new file mode 100755 index 0000000..90f889e --- /dev/null +++ b/llmware/models.py @@ -0,0 +1,16219 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The models module implements the model registry, the catalog for models and prompts, and classes that +implement the interface for each of the supported models. """ + +import os, logging, json, requests, tempfile, ast, time, shutil, importlib, sys, ctypes + +from collections import deque +from importlib import util +from typing import Mapping, Any +from pathlib import Path + +from llmware.util import Utilities, AgentWriter, LocalTokenizer +from llmware.configs import (LLMWareConfig, LLMWareException, ModelNotFoundException, + GGUFLibNotLoadedException,DependencyNotInstalledException) + +from llmware.model_configs import (global_model_repo_catalog_list, global_model_finetuning_prompt_wrappers_lookup, + global_default_prompt_catalog, model_benchmark_data, global_tokenizer_bos_eos_lookup) + +from llmware.gguf_configs import * +from llmware.gguf_configs import _LlamaModel, _LlamaContext, _LlamaBatch, _LlamaTokenDataArray + +# torch - import only if needed +# --torch is a required dependency for HFGenerativeModels and HFEmbeddingModels +# --if either of those classes is called, Torch will be imported at that time +torch = None +GLOBAL_TORCH_IMPORT = False + +# openvino - import only if needed +# --openvino and openvino_genai are dependencies of OVGenerativeModel +GLOBAL_OVG_IMPORT = False +GLOBAL_OPENVINO_IMPORT = False +ovg = None +openvino = None +ovc = None + +# onnxruntime_genai - import only if needed +# -- onnxruntime_genai is dependency of ONNXGenerativeModel +GLOBAL_ONNX_GENAI_RUNTIME = False +og = None + +# onnxruntime - import only if needed +# -- onnxruntime is dependency of ONNXEmbeddingModel +# -- it is called implicitly by ONNXGenerativeModel +GLOBAL_ONNX_CORE_RUNTIME = False +ort = None + +logger = logging.getLogger(__name__) +logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__)) + + +class _ModelRegistry: + + """ ModelRegistry class is wrapper class around the global_model_repo_catalog_list for easy dynamic updating, + and holds most of the key Model, ModelClass and Function/Tool mappings and configurations. """ + + # notes: + # --held out as internal global cls to keep options to adapt implementation over time + # --generally does not to be directly accessed -> make changes through ModelCatalog + + # pulls default model list from model_configs.py + registered_models = global_model_repo_catalog_list + + # global list of supported model classes with module lookup - and placeholder for other attributes over time + model_classes = {"ONNXGenerativeModel": {"module": "llmware.models", "open_source": True}, + "OVGenerativeModel": {"module": "llmware.models", "open_source": True}, + "GGUFGenerativeModel": {"module": "llmware.models", "open_source":True}, + "GGUFVisionGenerativeModel": {"module": "llmware.models", "open_source":True}, + "OVVisionGenerativeModel": {"module": "llmware.models", "open_source": True}, + "ONNXQNNGenerativeModel": {"module": "llmware.models", "open_source":True}, + "ONNXEmbeddingModel": {"module": "llmware.models", "open_source": True}, + "ONNXVisionGenerativeModel": {"module": "llmware.models", "open_source":True}, + "OVEmbeddingModel": {"module": "llmware.models", "open_source": True}, + "WindowsLocalFoundryModel": {"module": "llmware.models", "open_source":True}, + "WhisperCPPModel": {"module": "llmware.models", "open_source": True}, + "HFGenerativeModel": {"module": "llmware.models", "open_source":True}, + "HFReRankerModel": {"module": "llmware.models", "open_source": True}, + "LLMWareModel": {"module": "llmware.models", "open_source": True}, + "LLMWareSemanticModel": {"module": "llmware.models", "open_source": True}, + "HFEmbeddingModel": {"module": "llmware.models", "open_source": True}, + "OpenChatModel": {"module": "llmware.models", "open_source": True}, + "OllamaModel":{"module": "llmware.models", "open_source": True}, + "OpenAIGenModel":{"module": "llmware.models", "open_source": False}, + "ClaudeModel":{"module": "llmware.models", "open_source": False}, + "GoogleGeminiModel":{"module": "llmware.models", "open_source": False}, + "OpenAIEmbeddingModel":{"module": "llmware.models", "open_source": False}, + } + + model_catalog_state_attributes = ["selected_model", "loaded_model_name", "loaded_model_class", "temperature", + "api_endpoint", "get_logits", "max_output", "sample", + "force_reload", "account_name", "library_name", "api_key"] + + # model card validation for registering new model - required attributes + min_required_fields = ["model_name", "model_family", "model_category"] + + # most fine-tuned models require a specific prompt wrapping that was used in the fine-tuning process + # we are treating these "prompt_wrappers" as core attributes of the model + prompt_wrappers = ["alpaca", "human_bot", "chatgpt", "", "open_chat", "hf_chat", "chat_ml", "phi_3", + "llama_3_chat","tiny_llama_chat","stablelm_zephyr_chat", "google_gemma_chat", + "vicuna_chat", "phi_4", "deepseek_chat", "phi-4-mini", + "granite_chat", "lfm2_chat", "olmo_chat", "oss_chat", "phi_3_vision"] + + registered_wrappers = global_model_finetuning_prompt_wrappers_lookup + + # new attribute - track bos/eos for common tokenizers + tokenizer_bos_eos_config = global_tokenizer_bos_eos_lookup + + # list of specialized function calling tools + + llm_fx_tools = ["ner", "sentiment", "topics", "ratings", "emotions", "nli", + "intent", "sql", "answer", "category", "tags", "summary", "xsum", "extract", + "boolean", "sa-ner","tags-3b", "q_gen", "qa_gen"] + + llm_fx_tools_map = {"ner": "slim-ner-tool", + "sentiment": "slim-sentiment-tool", + "topics": "slim-topics-tool", + "ratings": "slim-ratings-tool", + "emotions": "slim-emotions-tool", + "nli": "slim-nli-tool", + "sql": "slim-sql-tool", + "tags": "slim-tags-tool", + "answer": "bling-answer-tool", + "category": "slim-category-tool", + "intent": "slim-intent-tool", + "summary": "slim-summary-tool", + "xsum": "slim-xsum-tool", + "extract": "slim-extract-tool", + "boolean": "slim-boolean-tool", + "sa-ner": "slim-sa-ner-tool", + "tags-3b": "slim-tags-3b-tool", + "q_gen": "slim-q-gen-tiny-tool", + "qa_gen": "slim-qa-gen-tiny-tool" + } + + _foundry_manager = None + + @classmethod + def get_model_list(cls): + """ List current view of registered models """ + return cls.registered_models + + @classmethod + def get_model_classes(cls): + """ List of model classes supported in LLMWare. """ + return cls.model_classes + + @classmethod + def add_model_class(cls, new_class, module="llmware.models", open_source=False,over_write=False): + + """ Adds a new model with flexibility to instantiate in new module. By default, it + assumes that the module is the current one, e.g., 'llmware.models'. """ + + if over_write or new_class not in cls.model_classes: + cls.model_classes.update({new_class:{"module": module, "open_source": open_source}}) + elif new_class in cls.model_classes: + logger.warning(f"_ModelRegistry: this model class - {new_class} already exists - to reset the module," + f"then please pass option over_write=True") + + @classmethod + def get_wrapper_list(cls): + """ List current registered wrapper formats """ + return cls.registered_wrappers + + # new method + @classmethod + def get_tokenizer_bos_eos_lookup(cls): + return cls.tokenizer_bos_eos_config + + @classmethod + def get_llm_fx_tools_list (cls): + """ List of function calling model tools available """ + return cls.llm_fx_tools + + @classmethod + def get_llm_fx_mapping (cls): + """ List of function calling model tools to repo name """ + return cls.llm_fx_tools_map + + @classmethod + def add_wrapper(cls, wrapper_name, wrapper_dict): + + """ Adds a new prompter wrapper to the registered list """ + + cls.registered_wrappers.update({wrapper_name:wrapper_dict}) + cls.prompt_wrappers.append(wrapper_name) + + return wrapper_dict + + @classmethod + def load_prompt_wrappers_from_file(cls, new_wrapper_registry): + + cls.registered_wrappers = {} + cls.prompt_wrappers = [] + + for key,value in new_wrapper_registry.items(): + if key not in cls.prompt_wrappers: + cls.prompt_wrappers.append(key) + + cls.registered_wrappers.update({key:value}) + + @classmethod + def load_tokenizer_configs_from_file(cls, new_tokenizer_configs): + + cls.tokenizer_bos_eos_config = {} + for key, value in new_tokenizer_configs.items(): + cls.tokenizer_bos_eos_config.update({key:value}) + + @classmethod + def validate(cls, model_card_dict): + + """ Provides minimal validation of structure of a new model card """ + + for keys in cls.min_required_fields: + if keys not in model_card_dict: + return False + + if "model_family" not in model_card_dict: + return False + + # removing this condition from validation - provides more extensibility in creating new model classes + """ + if model_card_dict["model_family"] not in cls.model_classes: + return False + """ + + if "prompt_wrapper" in model_card_dict: + + pwrap = model_card_dict["prompt_wrapper"] + + if pwrap: + + # ok if prompt_wrapper = "" + + if pwrap not in cls.get_wrapper_list(): + + # permits registering of new model card but issues warning + + logger.warning(f"this prompt wrapper - {pwrap} - is not registered which may lead " + f"to unpredictable results in inference - you should register this prompt " + f"format for better results.") + + return True + + @classmethod + def add_model(cls, model_card_dict, over_write=True): + + """ Adds a model to the registry """ + + if cls.validate(model_card_dict): + + # confirm that no overlap in names with model already in the catalog + + for i, model in enumerate(cls.registered_models): + if (model["model_name"] in [model_card_dict["model_name"], model_card_dict["display_name"]] or + model["display_name"] in [model_card_dict["model_name"], model_card_dict["display_name"]]): + + if not over_write: + + raise LLMWareException(message=f"Exception: model name overlaps with another model already " + f"in the ModelCatalog - {model}") + + else: + # logger.warning(f"_ModelRegistry - over-write = True - {model['model_name']} - mew model added.") + + del cls.registered_models[i] + + # go ahead and add model to the catalog + + cls.registered_models.append(model_card_dict) + + else: + raise LLMWareException(message="New Model Card is Missing Keys") + + return model_card_dict + + @classmethod + def update_model(cls, model_name_lookup, new_model_card_dict): + + """ Updates model in the registry """ + + if not cls.validate(new_model_card_dict): + raise LLMWareException(message="New Model Card is missing keys.") + + updated=False + + for i, models in enumerate(cls.registered_models): + # added option to match with display name + if models["model_name"] == model_name_lookup or models["display_name"] == model_name_lookup: + del cls.registered_models[i] + cls.registered_models.append(new_model_card_dict) + updated = True + break + + return updated + + @classmethod + def delete_model(cls, model_name): + + """ Removes model from Model Registry list """ + + model_found=False + + for i, models in enumerate(cls.registered_models): + # added option to match with display name + if models["model_name"] == model_name or models["display_name"] == model_name: + del cls.registered_models[i] + model_found = True + break + + if not model_found: + raise ModelNotFoundException(model_name) + + return model_found + + @classmethod + def new_model_registry(cls, model_registry): + + # remove current models + cls.registered_models = [] + + # add new model registry + for i, model in enumerate(model_registry): + + if cls.validate(model): + cls.registered_models.append(model) + + return True + + @classmethod + def get_model_catalog_vars(cls): + return cls.model_catalog_state_attributes + + @classmethod + def add_model_catalog_vars(cls, new_attr): + cls.model_catalog_state_attributes.append(new_attr) + return True + + @classmethod + def reset_to_default_catalog(cls): + cls.registered_models = global_model_repo_catalog_list + + + @classmethod + def get_foundry_manager(cls): + return cls._foundry_manager + + @classmethod + def reset_foundry_manager(cls): + cls._foundry_manager = None + return True + + @classmethod + def set_foundry_manager(cls, mgr): + cls._foundry_manager = mgr + return mgr + + @classmethod + def create_new_foundry_manager(cls): + from foundry_local import FoundryLocalManager + cls._foundry_manager = FoundryLocalManager() + return cls._foundry_manager + + +def pull_model_from_hf(model_card, local_model_repo_path, api_key=None, **kwargs): + + """ Fetches a specific model file from Huggingface repository into local model repo path, generally used for + GGUF models in a repository that contains multiple files - and this method will pull a single designated file. + + Inputs: model_card, path to the local model repo, and an api_key (optional). """ + + from huggingface_hub import hf_hub_download + + gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf", + gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf" + + if not os.path.exists(local_model_repo_path): + os.mkdir(local_model_repo_path) + + logger.warning(f"Models - pulling model from repo - {gguf_repo} - " + f"and will cache into local folder - {local_model_repo_path}") + + try: + downloader = hf_hub_download(gguf_repo, gguf_file, local_dir=local_model_repo_path, + local_dir_use_symlinks=False, token=api_key) + except: + raise LLMWareException(message=f"Models - load_model - pull_model_from_hf - Something has " + f"gone wrong in the download process. Please try again.") + + # remove ongoing links, if any, created by attributes not in the file repo + files_created = os.listdir(local_model_repo_path) + + if "validation_files" in model_card: + validation_files = model_card["validation_files"] + for files in validation_files: + if files not in files_created: + logger.warning(f"Models - load_model - pull_snapshot_from_hf - missing validation file " + f"expected to run the model correctly - {files}") + + if ".huggingface" in files_created: + try: + shutil.rmtree(os.path.join(local_model_repo_path,".huggingface")) + logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .huggingface") + except: + logger.info(f"Models - load_model - pull_snapshot_from_hf - " + f".huggingface folder created in repo and not auto-removed.") + pass + + if ".cache" in files_created: + try: + shutil.rmtree(os.path.join(local_model_repo_path,".cache")) + logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .cache") + except: + logger.info(f"Models - load_model - pull_snapshot_from_hf - " + f".cache folder created in repo and not auto-removed.") + pass + + if ".gitattributes" in files_created: + try: + os.remove(os.path.join(local_model_repo_path, ".gitattributes")) + logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .gitattributes") + except: + logger.info(f"Models - load_model - pull_snapshot_from_hf - " + f".gitattributes created in repo and not auto-removed.") + pass + + return local_model_repo_path + + +def pull_snapshot_from_hf(model_card, local_model_repo_path, api_key=None, **kwargs): + + """ Fetches snapshot of HF model repository and saves into local folder path - two required + inputs: + -- repo_name - the full name of the Huggingface repo, e.g., microsoft/phi-2 + -- local_model_repo_path - the local path to save the model files. + """ + + from huggingface_hub import snapshot_download + + if "gguf_repo" in model_card: + repo_name = model_card["gguf_repo"] + elif "hf_repo" in model_card: + repo_name = model_card["hf_repo"] + elif "ov_repo" in model_card: + repo_name = model_card["ov_repo"] + else: + raise LLMWareException("Model Fetch process error: no repo identified as source to fetch the model.") + + # repo_name = model_card["gguf_repo"] + + try: + snapshot = snapshot_download(repo_name, local_dir=local_model_repo_path, token=api_key, + local_dir_use_symlinks=False) + except: + raise LLMWareException(message=f"Models - load_model - pull_snapshot_from_hf - {repo_name} - Something has " + f"gone wrong in the download process. Please try again.") + + files_created = os.listdir(local_model_repo_path) + + logger.debug(f"Models - load_model - pull_snapshot_from_hf - downloaded snapshot - " + f"files cached locally - {files_created}") + + if "validation_files" in model_card: + validation_files = model_card["validation_files"] + for files in validation_files: + if files not in files_created: + logger.warning(f"Models - load_model - pull_snapshot_from_hf - missing validation file " + f"expected to run the model correctly - {files}") + + # clean up any residual download artifacts in model folder + if ".huggingface" in files_created: + try: + shutil.rmtree(os.path.join(local_model_repo_path,".huggingface")) + logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .huggingface") + except: + logger.info(f"Models - load_model - pull_snapshot_from_hf - .huggingface folder created in " + f"repo and not auto-removed.") + pass + + if ".cache" in files_created: + try: + shutil.rmtree(os.path.join(local_model_repo_path,".cache")) + logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .cache") + except: + logger.info(f"Models - load_model - pull_snapshot_from_hf - " + f".cache folder created in repo and not auto-removed.") + pass + + if ".gitattributes" in files_created: + try: + os.remove(os.path.join(local_model_repo_path, ".gitattributes")) + logger.debug("Models - load_model - pull_snapshot_from_hf - removed: .gitattributes") + except: + logger.info(f"Models - load_model - pull_snapshot_from_hf - .gitattributes created " + f"in repo and not auto-removed.") + pass + + return local_model_repo_path + + +class ModelCatalog: + + """ ModelCatalog is the main class responsible for model lookup of (1) Model Card and (2) Finding Model Class. + In most cases, ModelCatalog is the interface for all facets of interacting with the model classes. + """ + + def __init__(self): + + # ModelCatalog is simple, flexible mechanism to track registered models + # Easy to create "model repo" with mix of model types and instantiation approaches + # Builds on standard model classes with standard inference + + self.model_classes = _ModelRegistry().get_model_classes() + + self.global_model_list = _ModelRegistry().get_model_list() + + self.base_attributes = _ModelRegistry().get_model_catalog_vars() + + self.account_name = None + self.library_name= None + + # attributes that are used when a model is selected through .load_model method + self.loaded_model_name = None + self.loaded_model_class = None + self.temperature = 0.3 + self.use_gpu = True + self.sample = True + self.max_output = 100 + self.get_logits = False + self.force_reload = False + self.api_endpoint = None + + self.selected_model = None + self.api_key= None + self.custom_loader = None + + # new - add - 102024 + self.model_kwargs = {} + + def to_state_dict(self): + + """ Writes selected model state parameters to dictionary. """ + + state_dict = {} + for keys in self.base_attributes: + if hasattr(self, keys): + state_dict.update({keys: getattr(self, keys)}) + + return state_dict + + def pull_latest_manifest(self): + """ Not implemented currently """ + # will add to check manifest in global repo and make available for pull down + return 0 + + def reset_to_default_catalog(self): + """ Resets model catalog to default list in model_configs """ + + _ModelRegistry().reset_to_default_catalog() + self.global_model_list = _ModelRegistry().get_model_list() + + def save_model_registry(self, fp=None, fn="llmware_model_catalog.json"): + + """ Utility method to export global model list to json file """ + + if not fp: + fp = LLMWareConfig().get_model_repo_path() + + json_dict = json.dumps(self.global_model_list, indent=1) + with open(os.path.join(fp, fn), "w", encoding='utf-8') as outfile: + outfile.write(json_dict) + + return 0 + + def load_model_registry(self, fp=None, fn="llmware_model_catalog.json"): + + """ Utility method to load global model list from json file. Will remove the current + global model list and replace with the model cards from file. """ + + if not fp: + fp = LLMWareConfig().get_model_repo_path() + + model_list = json.load(open(os.path.join(fp,fn), "r")) + + _ModelRegistry().new_model_registry(model_list) + + self.global_model_list = _ModelRegistry().get_model_list() + + return 0 + + def load_prompt_wrapper_registry(self, fp=None, fn="prompt_wrappers.json"): + + """ Utility method to load updated prompt wrapper registry from json file. Will + remove the current global prompt wrapper registry and replace with updated registry from file. """ + + if not fp: + fp = LLMWareConfig().get_llmware_path() + + prompt_list = json.load(open(os.path.join(fp,fn), "r")) + _ModelRegistry().load_prompt_wrappers_from_file(prompt_list) + + return True + + def save_prompt_wrapper_registry(self, fp=None, fn="prompt_wrappers.json"): + + """ Utility method to export global prompt wrapper list to json file """ + + if not fp: + fp = LLMWareConfig().get_llmware_path() + + prompt_list = _ModelRegistry().get_wrapper_list() + + json_dict = json.dumps(prompt_list, indent=1) + with open(os.path.join(fp, fn), "w", encoding='utf-8') as outfile: + outfile.write(json_dict) + + return True + + def get_tokenizer_bos_eos_configs(self): + + """" Returns the tokenizer bos eos configs for common models. """ + + return _ModelRegistry().get_tokenizer_bos_eos_lookup() + + def save_tokenizer_bos_eos_configs(self, fp=None, fn="tokenizer_bos_eos_configs.json"): + + """ Utility method to export tokenizer bos_eos configs to json file """ + + if not fp: + fp = LLMWareConfig().get_llmware_path() + + tok_configs = _ModelRegistry().get_tokenizer_bos_eos_lookup() + + json_dict = json.dumps(tok_configs, indent=1) + with open(os.path.join(fp, fn), "w", encoding='utf-8') as outfile: + outfile.write(json_dict) + + return True + + def load_tokenizer_bos_eos_configs(self, fp=None, fn="tokenizer_bos_eos_configs.json"): + + """ Utility method to load updated tokenizer bos_eos configs from json file. Will + remove the current tokenizer bos eos configs and replace with updated configs from file. """ + + if not fp: + fp = LLMWareConfig().get_llmware_path() + + tok_config_list = json.load(open(os.path.join(fp, fn), "r")) + _ModelRegistry().load_tokenizer_configs_from_file(tok_config_list) + + return True + + def add_model_cards_from_file(self, fp=None, fn="custom_models_manifest.json"): + + """ Utility method that loads model cards from a single json file and incrementally adds + to the model global model list. """ + + if not fp: + fp = LLMWareConfig().get_model_repo_path() + + model_add_list = json.load(open(os.path.join(fp, fn), "r")) + + for i, model in enumerate(model_add_list): + _ModelRegistry().add_model(model) + + self.global_model_list = _ModelRegistry().get_model_list() + + return 0 + + def register_new_model_card(self, model_card_dict): + + """ Registers a new model card directly in the model catalog """ + + _ModelRegistry().add_model(model_card_dict) + + # update the global list in ModelCatalog instance + self.global_model_list = _ModelRegistry().get_model_list() + + return 0 + + def delete_model_card(self, model_name): + + """ Removes a model card from the registry """ + + _ModelRegistry().delete_model(model_name) + + # update current ModelCatalog instance + self.global_model_list = _ModelRegistry().get_model_list() + + return 0 + + def register_new_finetune_wrapper(self, name, main_start="", main_stop="", llm_start="", + system_start="", system_stop=""): + + """ Registers a new fine-tuning wrapper using a basic template that assembles a prompt and will add + special tokens as indicated in the wrapper: + + -- main_start - token, if any, to be provided at the start of the prompt template + -- main_stop - token, if any, to be provided at the end of the main 'user' input + -- llm_start - token, if any, at the end of the prompt that is the signal to start the 'assistant' role + -- system_start - optional token to start an initial segment indicating a 'system' instruction + -- system_stop - optional token to stop an initial segment indicating a 'system' instruction. + + For example, the LLama-2-Chat wrapper is implemented as follows: + + main_start = "" + main_stop = " + llm_start = "" + + """ + + new_dict = {"main_start": main_start, "main_stop": main_stop, "start_llm_response": llm_start, + "system_start": system_start, "system_stop": system_stop} + + _ModelRegistry().add_wrapper(name, new_dict) + + return 0 + + def get_list_registered_finetune_wrappers(self): + + """ Returns an updated list of registered fine-tuning wrappers. """ + + return _ModelRegistry().get_wrapper_list() + + def register_new_hf_generative_model(self, hf_model_name, llmware_lookup_name=None, display_name=None, + context_window=2048, prompt_wrapper="", + temperature=0.3, trailing_space="", link=""): + + """ Registers any Huggingface Generative Model in the ModelCatalog for easy future lookup and + integration into LLMWare RAG workflows. + + The most important input parameter is hf_model_name, which should correspond to the Huggingface Repo/Model + format, e.g., microsoft/phi-2 + + Any names can be assigned as 'aliases' for the LLMWare Model catalog with both a main lookup name and an + optional secondary lookup to be used as a short-name for screen display. + + For example, the 'llmware_lookup_name' for 'microsoft/phi-2' could be 'phi-2' + or 'my-favorite-model-with-2-in-the-name'. + + If no llmware_lookup_name is provided, then it will automatically save as the hf_model_name. """ + + if not llmware_lookup_name: + llmware_lookup_name = hf_model_name + + if not display_name: + display_name = hf_model_name + + model_card = {"model_name": llmware_lookup_name, + "context_window": context_window, + "prompt_wrapper": prompt_wrapper, + + # hf_model_name should correspond to the hf repo/model standard + "hf_repo": hf_model_name, + "display_name": display_name, "temperature": temperature, "trailing_space": trailing_space, + "model_family": "HFGenerativeModel", "model_category": "generative_local", + "model_location": "hf_repo", "instruction_following": False, + "link": link, + "custom_model_files": [], "custom_model_repo": ""} + + _ModelRegistry().add_model(model_card) + + self.global_model_list = _ModelRegistry().get_model_list() + + return model_card + + def register_sentence_transformer_model(self, model_name, embedding_dims, context_window, + display_name=None, link=""): + + """ Registers a model from the SentenceTransformers library into an LLMWare Model Catalog. + + NOTE: for SentenceTransformers, the model_name should match the SentenceTransformer library lookup + name. """ + + if not display_name: + display_name = model_name + + new_model_card_dict = {"model_name": model_name, "context_window": context_window, + "embedding_dims": embedding_dims, + # pre-populated parameters for sentence transformer + "model_family": "LLMWareSemanticModel", "model_category": "embedding", + "display_name": display_name, "link": link, + "model_location": "st_repo", + "custom_model_files": [], "custom_model_repo":"" + } + + _ModelRegistry().add_model(new_model_card_dict) + + self.global_model_list = _ModelRegistry().get_model_list() + + return new_model_card_dict + + def register_gguf_model(self, model_name, gguf_model_repo, gguf_model_file_name, prompt_wrapper=None, + eos_token_id=0, display_name=None,trailing_space="", temperature=0.3, + context_window=2048, instruction_following=True): + + """ Registers a new GGUF model in model catalog - by default, assumes that the GGUF file is in a Huggingface + repository, and will be pulled directly from that repository into a local model_repo cache. + + Any arbitrary name can be selected as the model_name and/or display_name for the llmware catalog, as the + core lookup is in the "gguf_repo" and "gguf_file" parameters. + + If the GGUF file is in another local file path, then you can access it directly by setting: + + "custom_model_repo": "/path/to/local/gguf_model/" + "custom_model_files": "my_model.gguf" + + """ + + if not display_name: + display_name = model_name + + new_model_card_dict = {"model_name": model_name, "display_name": display_name, + "model_family": "GGUFGenerativeModel", "model_category": "generative_local", + "model_location": "llmware_repo", "context_window": context_window, + "instruction_following": instruction_following, "prompt_wrapper": prompt_wrapper, + "temperature": temperature, "trailing_space": trailing_space, + "eos_token_id": eos_token_id, + "gguf_file": gguf_model_file_name, + "gguf_repo": gguf_model_repo, + "link": "", "custom_model_files": [], "custom_model_repo":"", + "fetch": {"module":"llmware.models","method":"pull_model_from_hf"}, + "validation_files":[gguf_model_file_name] + } + + _ModelRegistry().add_model(new_model_card_dict) + + self.global_model_list = _ModelRegistry().get_model_list() + + return new_model_card_dict + + def register_open_chat_model(self, model_name, api_base=None, model_type="chat", display_name=None, + context_window=4096, instruction_following=True, prompt_wrapper="", + temperature=0.5): + + """ Add any open chat model into the LLMWare Model Catalog for easy access, e.g., + + ModelCatalog().register_open_chat_model("my_open_chat_model1", api_base="http://localhost:1234/v1", + prompt_wrapper="", model_type="chat") + + To invoke the model: + + my_open_chat_model = ModelCatalog().load_model("my_open_chat_model1") + + Or from a prompt: + + prompter = Prompt().load_model("my_open_chat_model1") + + """ + + if not display_name: + display_name = model_name + + new_model_card_dict = {"model_name": model_name, "model_type": model_type, "prompt_wrapper": prompt_wrapper, + "display_name": display_name, + "model_family": "OpenChatModel", "model_category": "generative-api", + "model_location": "api", "context_window": context_window, + "instruction_following": instruction_following, + "temperature": temperature, "trailing_space": "", + "api_base": api_base + } + + _ModelRegistry().add_model(new_model_card_dict) + + self.global_model_list = _ModelRegistry().get_model_list() + + return 0 + + def register_ollama_model(self, model_name, host="localhost", port=11434, model_type="chat", + raw=False, stream=False, display_name=None, context_window=4096, + instruction_following=True, prompt_wrapper="", temperature=0.5): + + """ Add any Ollama model into Model Catalog - key parameters: + + Assumes - + 1. default host/port configs of "localhost:11434" + 2. supports 'completion' ollama api, but uses "chat" by default + 3. assumes raw=False & stream=False -> more options will be supported over time + + If you are using the ollama default settings, then you can register a model card by + simply providing the model name, + + e.g., ModelCatalog().register_ollama_model("llama2") + + """ + + if not display_name: + display_name = model_name + + # note: both raw_mode and stream_mode are set to False + + new_model_card_dict = {"model_name": model_name, "model_type": model_type, + "host": host, "port": port, + "prompt_wrapper": prompt_wrapper, + "display_name": display_name, + "model_family": "OllamaModel", "model_category": "generative-api", + "model_location": "api", "context_window": context_window, + "instruction_following": instruction_following, + "temperature": temperature, "trailing_space": "", + "raw_mode": False, "stream_mode": False + } + + _ModelRegistry().add_model(new_model_card_dict) + + self.global_model_list = _ModelRegistry().get_model_list() + + return 0 + + def setup_custom_llmware_inference_server(self, uri_string, secret_key=None): + + """ Sets up and registers a custom llmware inference server """ + + # Examples: + # os.environ["LLMWARE_GPT_URI"] = "http://111.111.1.111:8080" + # os.environ["USER_MANAGED_LLMWARE_GPT_API_KEY"] = "demo-pass-test-key" + + # set environ variables with the URL and password key + os.environ["LLMWARE_GPT_URI"] = uri_string + os.environ["USER_MANAGED_LLMWARE_GPT_API_KEY"] = secret_key + + return 1 + + def lookup_model_card (self, selected_model_name): + + """ Looks up a model card by model name - the model card has the key configuration and lookup information """ + + model_card = None + + # first check in the global_model_repo + confirm location + for models in self.global_model_list: + + # add option to match with display_name as alternative alias for model + if models["model_name"] == selected_model_name or models["display_name"] == selected_model_name: + model_card = models + model_card.update({"standard":True}) + break + + # if model not found, then return None, and downstream calling function responsible for handling + + return model_card + + def _instantiate_model_class_from_string(self, model_class, model_name, model_card, api_key=None, + api_endpoint=None, **kwargs): + + """ Internal utility method to instantiate model classes from strings. """ + + # by default - if model not found - return None + my_model = None + context_window= 2048 # used in generative models - use 2048 as default safe backup + embedding_dims = None # used in embedding models + + if "context_window" in model_card: + context_window = model_card["context_window"] + + if "embedding_dims" in model_card: + embedding_dims = model_card["embedding_dims"] + + if model_class in self.model_classes: + + module = self.model_classes[model_class]["module"] + model_module = importlib.import_module(module) + if hasattr(model_module, model_class): + model_class = getattr(model_module, model_class) + + my_model = model_class(model_name=model_name, context_window=context_window, + api_key=api_key, + trust_remote_code=True, + model_card=model_card, + use_gpu_if_available=self.use_gpu, + get_logits=self.get_logits, + temperature=self.temperature, + max_output=self.max_output, + sample=self.sample, + embedding_dims=embedding_dims, + api_endpoint=api_endpoint, + **kwargs) + else: + raise LLMWareException(message=f"Exception: {model_class} not found.") + + return my_model + + def model_load_optimizer(self): + + """ Enables the ability to intercept the standard model loading process for inserting 'auto optimization' + steps, such as the availability of an API instance of the model or a better performing package, e.g., GGUF + given the intended deployment environment, or even a preferred implementation/version of the model - + without having to change any code. + + Currently, not implemented by default, but can be configured to enable custom steps to enable + advanced model routing optimization. """ + + router_method = "" + router_class = "" + exec_method = None + + model_router = LLMWareConfig().get_config("model_router") + router_module = model_router["module"] + if "class" in model_router: + router_class = model_router["class"] + if "method" in model_router: + router_method = model_router["method"] + + module = importlib.import_module(router_module) + + if router_class: + if hasattr(module, router_class): + exec_class = getattr(module, router_class)() + if hasattr(exec_class, router_method): + exec_method = getattr(exec_class, router_method) + else: + if hasattr(module, router_method): + exec_method = getattr(module, router_method) + + if exec_method: + success_dict = exec_method(self.to_state_dict()) + if success_dict: + # write attributes, if any, to the ModelCatalog state, which will be picked up + # to "re-direct" the model loading parameters + if isinstance(success_dict, dict): + for k, v in success_dict.items(): + setattr(self,k,v) + + return True + + def load_model (self, selected_model, api_key=None, use_gpu=True, sample=True,get_logits=False, + max_output=100, temperature=-99, force_reload=False, api_endpoint=None, + custom_loader=None, **kwargs): + + """ Main method for loading and fully instantiating a model with lookup based on the model_name in + the ModelCatalog. """ + + # apply optional attributes - will be available to the loaded model + self.use_gpu=use_gpu + self.sample=sample + self.max_output=max_output + self.get_logits=get_logits + self.force_reload = force_reload + self.api_endpoint = api_endpoint + + self.selected_model = selected_model + self.api_key=api_key + self.use_gpu = use_gpu + self.custom_loader = custom_loader + + # note: temperature set by default at -99, which is a dummy value that is over-ridden by the temperature + # in the model card. This temperature will only be used if explicitly set by the user at value != -99 + + self.temperature=temperature + + # assumed to be set to FALSE in default configs - should not be changed until model route optimizer implemented + if LLMWareConfig().get_config("apply_model_load_router"): + self.model_load_optimizer() + + # completes all preparatory steps, and returns 'ready-for-inference' model + selected_model = self.selected_model + + logger.debug(f"ModelCatalog - load_model - loading model - {selected_model}") + + # step 1- lookup model card from the catalog + model_card = self.lookup_model_card(self.selected_model) + if not model_card: + logger.error(f"error: ModelCatalog - unexpected - could not identify model card for " + f"selected model - {self.selected_model}") + + raise ModelNotFoundException(self.selected_model) + + # new - 1020 add + if self.model_kwargs: + if not kwargs: + kwargs = {} + for k,v in self.model_kwargs.items(): + kwargs.update({k:v}) + # end - new add + + # step 2- instantiate the right model class + my_model = self.get_model_by_name(model_card["model_name"], api_key=self.api_key, + api_endpoint=self.api_endpoint, **kwargs) + + if not my_model: + logger.error(f"error: ModelCatalog - unexpected - could not identify the model - " + f"{self.selected_model}") + + raise ModelNotFoundException(self.selected_model) + + # step 3- if physical model, then need to locate, validate, potentially fetch and then load + + if model_card["model_location"] == "llmware_repo" and not self.api_endpoint: + + loading_directions = self.prepare_local_model(model_card, + custom_loader=self.custom_loader, + api_key=self.api_key, + **kwargs) + + my_model = my_model.load_model_for_inference(loading_directions, model_card=model_card, **kwargs) + + else: + # if api_key passed, save as environ variable + # TODO - look at this + if api_key: + my_model.set_api_key(api_key) + os.environ[selected_model] = api_key + + # pass model name to the model directly + my_model.model_name = selected_model + + return my_model + + def prepare_local_model(self, model_card, custom_loader=None, api_key=None, **kwargs): + + """ Resolves obtaining a valid local path to the required model components. + + 1. Identify if model is available in local path. + -- if custom path provided, then validate from that path. + -- if custom loader provided, then use custom loader to complete this step + -- once local path resolved: + -- Validate that local path contains the required elements + -- Return the loading path to load_the_model_for_inference + + 2. If not available locally, then need to fetch. + -- Use the fetch method provided in the Model Card + -- if not provided, then use a default for model class + -- need to provide error-handling if download fails + + """ + + # Step 1 - resolve local path + + if custom_loader: + return custom_loader(model_card, api_key=api_key) + + if "custom_model_repo" in model_card: + custom_repo = model_card["custom_model_repo"] + else: + custom_repo = None + + if custom_repo and os.path.exists(custom_repo): + + # if path exists ... (if null result, then will continue down main resolve path) + + custom_local_path = self.check_custom_local_repo(model_card, api_key=api_key) + if custom_local_path: + return custom_local_path + + # Main resolve path + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + if not os.path.exists(LLMWareConfig.get_model_repo_path()): + os.mkdir(LLMWareConfig.get_model_repo_path()) + + # strip '/' from model name + model_folder_name = model_card["model_name"].split("/")[-1] + + model_location = os.path.join(LLMWareConfig.get_model_repo_path(), model_folder_name) + + go_ahead = False + + if os.path.exists(model_location): + + go_ahead = True + + model_files = os.listdir(model_location) + + if "validation_files" in model_card: + for file in model_card["validation_files"]: + if file not in model_files: + go_ahead = False + break + + if len(model_files) == 0: + go_ahead = False + + if go_ahead: + return model_location + + if not go_ahead: + + # need to fetch the model files + + fetch, fetch_method_name = self.fetch_resolve(model_card) + + if fetch and fetch_method_name: + + logger.warning(f"ModelCatalog - load_model - fetching model - {model_card['model_name']} - " + f"from remote repository using {fetch_method_name} - " + f"this may take a couple of minutes the first time.") + + # fetch method input: model_card, save_to_path, api_key (optional) + # fetch method must be able to resolve the repo using info in the model card + success = fetch(model_card, model_location, api_key=api_key, **kwargs) + + if isinstance(success, dict): + # write attributes, if any, to the Model instance state + for k, v in success.items(): + setattr(self, k, v) + + return model_location + + else: + raise(LLMWareException(message=f"Models - load_model - selected model not found in local path - and " + f"could not identify a supporting fetch method to " + f"retrieve selected model from model repository.")) + + def fetch_resolve(self, model_card): + + """ Returns the fetch method from model card - if not found, then loads default. """ + + # need to fetch the model -> will use fetch method provided in model card + fetch_module = None + fetch_method = None + fetch_class = None + fetch_exec = None + + default_fetch = LLMWareConfig().get_config("model_fetch") + + if LLMWareConfig().get_config("apply_default_fetch_override"): + + # if set to True, will over-ride the model card and use the default fetch mechanism + + fetch_module = default_fetch["module"] + if "class" in default_fetch: + fetch_class = default_fetch["class"] + if "method" in default_fetch: + fetch_method = default_fetch["method"] + + else: + + # primary (default) case - each model card provides configs for how to fetch the model + + if "fetch" in model_card: + if "module" in model_card["fetch"]: + fetch_module = model_card["fetch"]["module"] + if "method" in model_card["fetch"]: + fetch_method = model_card["fetch"]["method"] + if "class" in model_card["fetch"]: + fetch_class = model_card["fetch"]["class"] + + if not fetch_module: + + # fallback case - if not provided in model card, then fallback to the default fetch mechanism + + fetch_module = default_fetch["module"] + + if "class" in default_fetch: + fetch_class = default_fetch["class"] + if "method" in default_fetch: + fetch_method = default_fetch["method"] + + module = importlib.import_module(fetch_module) + + if fetch_class: + if hasattr(module, fetch_class): + class_exec = getattr(module, fetch_class)() + if hasattr(class_exec, fetch_method): + fetch_exec = getattr(class_exec,fetch_method) + else: + if hasattr(module, fetch_method): + fetch_exec = getattr(module, fetch_method) + + return fetch_exec, fetch_method + + def check_custom_local_repo(self, model_card, api_key=None): + + """ Model card provides the option for a custom local path as the execution location for the model. + If 'custom_model_repo' parameter found, then this method will resolve the local path and return + that local path for loading the model. """ + + # if custom model repo path provided in model card, then pull model from this path + if "custom_model_repo" in model_card: + if model_card["custom_model_repo"]: + if os.path.exists(model_card["custom_model_repo"]): + if "custom_model_files" in model_card: + if model_card["custom_model_files"]: + if len(model_card["custom_model_files"]) > 0: + if os.path.exists(os.path.join(model_card["custom_model_repo"], + model_card["custom_model_files"][0])): + + # confirmed that custom path and at least model artifact exist + logger.info(f"update: returning custom model path: " + f"{model_card['custom_model_repo']} - " + f"{model_card['custom_model_files']}") + + return model_card["custom_model_repo"] + else: + raise ModelNotFoundException(f"Custom model repo path - {model_card['custom_model_repo']}") + + # fallback - if can not validate the path, then will return None and handle in caller + + return None + + def add_api_key (self, selected_model_name, api_key): + + """ Convenience method to apply an api_key to a pass to a model """ + + # step 1- lookup model card from the catalog + model_card = self.lookup_model_card(selected_model_name) + + if not model_card: + + logger.error(f"error: ModelCatalog - could not identify model card for " + f"selected model - {selected_model_name}") + + raise ModelNotFoundException(selected_model_name) + + # step 2 - save api key as environmental variable + model_name = model_card["model_name"] + os.environ[model_name] = api_key + + return self + + def load_sentence_transformer_model(self,model, model_name): + + """ Loads a sentence transformer model """ + + model = LLMWareSemanticModel(model=model,model_name=model_name) + return model + + def load_hf_embedding_model(self, model, tokenizer,trust_remote_code=False): + + """ Loads and integrates a Huggingface embedding model """ + + model = HFEmbeddingModel(model, tokenizer, trust_remote_code=trust_remote_code) + return model + + def load_hf_generative_model(self, model,tokenizer,prompt_wrapper=None, + instruction_following=False): + + """ Loads and integrates a Huggingface generative decoder-based 'causal' model with limited options + to control model preprocessing prompt behavior """ + + model = HFGenerativeModel(model, tokenizer, prompt_wrapper=prompt_wrapper, + instruction_following=instruction_following) + + return model + + def load_embedding_model (self, model_name=None, + model=None, tokenizer=None,from_hf=False, + from_sentence_transformers=False): + + """ Loads embedding model by name - + main handler used by any calling function to instantiate embedding model. """ + + loaded_model = None + + # if user passed a 'loaded model' object, then apply directly + if model: + + # first, check for 'from_hf' flag and load as HuggingFace model + if from_hf: + loaded_model = ModelCatalog().load_hf_embedding_model(model,tokenizer, trust_remote_code=True) + else: + # second, check for 'from_sentence_transformer' flag and load as SBERT model + if from_sentence_transformers: + loaded_model = ModelCatalog().load_sentence_transformer_model(model,model_name) + + if not loaded_model: + logger.error("ModelCatalog - load_embedding_model - could not identify the " + "passed model - if model is from HuggingFace, then mark optional " + "'from_hf' flag to True. If model is from Sentence Transformers, " + "then mark optional 'from_sentence_transformers' flag " + "to True. Note: setting search mode to text search, in absence of embedding " + "model.") + else: + # main case - load embedding model from Catalog + loaded_model = ModelCatalog().load_model(selected_model=model_name) + + return loaded_model + + def list_open_source_models(self): + + """ Lists the open source models in the ModelCatalog. """ + + open_source_models = [] + + open_source_class = [] + model_classes = _ModelRegistry().get_model_classes() + for key, value in model_classes.items(): + if "open_source" in value: + if value["open_source"]: + open_source_class.append(key) + + for x in self.global_model_list: + + if x["model_family"] in open_source_class: + open_source_models.append(x) + + return open_source_models + + def list_models_by_type(self, model_family): + + model_list = [] + + # e.g., model_family = "WindowsLocalFoundryModel" + + for model in self.global_model_list: + + if model["model_family"].lower() == model_family.lower(): + model_list.append(model) + + return model_list + + def list_embedding_models(self): + + """ Lists the embedding models in the ModelCatalog. """ + + embedding_models = [] + + for x in self.global_model_list: + if x["model_category"] == "embedding": + embedding_models.append(x) + + return embedding_models + + def list_generative_models(self): + + """ Lists the generative models in the ModelCatalog. """ + + gen_models = [] + + for x in self.global_model_list: + if x["model_category"].startswith("generative"): + gen_models.append(x) + + gen_models = sorted(gen_models, key=lambda x: x["model_name"], reverse=False) + + return gen_models + + def list_generative_local_models(self): + + """ Lists the generative local models in the ModelCatalog. """ + + gen_local_models = [] + + for x in self.global_model_list: + if x["model_category"] == "generative_local": + gen_local_models.append(x) + + gen_local_models = sorted(gen_local_models, key=lambda x:x["model_name"], reverse=False) + + return gen_local_models + + def list_all_models(self): + + """ Lists all models in the ModelCatalog. """ + + all_models = [] + for x in self.global_model_list: + all_models.append(x) + + all_models = sorted(all_models, key=lambda x: x["model_category"], reverse=False) + + return all_models + + def list_intel_npu_optimized_models(self): + + npu_models = [] + for model_card in self.global_model_list: + npu_optimized = model_card.get("npu_optimized","") + if npu_optimized: + npu_models.append(model_card) + + return npu_models + + def model_lookup(self,model_name): + + """ Looks up model by model_name. Will check both the primary 'model_name' and the secondary/optional + display_name to look for a match in the ModelCatalog. """ + + my_model = None + + for models in self.global_model_list: + # add check for match with display_name as alias + if models["model_name"] == model_name or models["display_name"] == model_name: + my_model = models + break + + return my_model + + def get_model_by_name(self, model_name, api_key=None, api_endpoint=None, **kwargs): + + """ Gets and instantiates model by name. """ + + my_model = None + + for models in self.global_model_list: + + # add check for display name match + if models["model_name"] == model_name or models["display_name"] == model_name: + selected_model = models + my_model = self._instantiate_model_class_from_string(selected_model["model_family"], + model_name, models,api_key=api_key, + api_endpoint=api_endpoint, **kwargs) + break + + return my_model + + def save_benchmark_report(self, fp=None,fn=None): + + """ Saves model benchmark score data to jsonl file. Optional inputs to assign folder path (fp) and + filename (fn). If not provided, then will be saved in llmware_data path with default name. + """ + + if not fp: + fp = LLMWareConfig().get_llmware_path() + + if not fn: + fn = "llmware_model_benchmark_scores" + + test_fn = fn + ".jsonl" + + f_out = open(os.path.join(fp, test_fn), "w") + + for entry in model_benchmark_data: + jsonl_row = json.dumps(entry) + f_out.write(jsonl_row) + f_out.write("\n") + + f_out.close() + + return fp + def get_benchmark_score(self, model_name): + + """ Looks up benchmark score for a model, if available. Returns None if no benchmark available. """ + + for i, entry in enumerate(model_benchmark_data): + if entry["model_name"] == model_name: + return entry + + logger.debug(f"ModelCatalog - get_benchmark_score - {model_name} does not have a benchmark available.") + + return None + + def get_benchmark_by_filter (self, conditions=None): + + """ Will apply a list of {key:value} conditions to provide a subset of models that fit the conditions. + + Conditions are a list of dictionaries, with each dictionary entry consisting of the following: + -- {key, "eval str"}, + -- e.g., {"parameters", "parameters < 3"} + + To create multiple conditions - create a list of several dictionaries: + -- e.g., [ {"parameters", "parameters < 6"}, {"accuracy_score", "accuracy_score > 95"} ] + """ + + if not conditions: + + logger.debug("ModelCatalog - get_benchmark_by_filter - no conditions provided, so returning all of the " + "benchmark data list.") + + return model_benchmark_data + + if isinstance(conditions,dict): + conditions = [conditions] + else: + if not isinstance(conditions,list): + logger.warning(f"ModelCatalog - conditions should be structured as a list of dictionary entries, " + f"with each dictionary entry consisting of a pair of a key:eval_str") + return model_benchmark_data + + results = [] + for i, entry in enumerate(model_benchmark_data): + + num_conditions = 0 + true_conditions = 0 + + for cond in conditions: + if isinstance(cond, dict): + num_conditions += 1 + for key,value in cond.items(): + if key in entry: + truth_value = eval(value, {key:entry[key]}) + if truth_value: + true_conditions += 1 + + if num_conditions > 0 and num_conditions == true_conditions: + results.append(entry) + + return results + + def get_llm_toolkit(self, tool_list=None, api_key=None): + + """ Caches all SLIM tools by default, or if list provided, then selected tools only. """ + + model_repo_path = LLMWareConfig.get_model_repo_path() + + if not os.path.exists(model_repo_path): + os.makedirs(model_repo_path) + + if not tool_list: + tool_list = _ModelRegistry().get_llm_fx_tools_list() + + for tool in tool_list: + + tool_name = _ModelRegistry().get_llm_fx_mapping()[tool] + + logger.info(f"ModelCatalog - get_toolset - {tool} - {tool_name}") + + found_model = False + local_model_repo_path = os.path.join(model_repo_path, tool_name) + + if os.path.exists(local_model_repo_path): + model_parts_in_folder = os.listdir(local_model_repo_path) + if len(model_parts_in_folder) > 0: + found_model = True + + if not found_model: + + model_card = self.lookup_model_card(tool_name) + pull_snapshot_from_hf(model_card, local_model_repo_path, api_key=api_key) + + return 0 + + def list_llm_tools(self): + + """Provides a list of the currently available SLIM tools available in the catalog. """ + + return _ModelRegistry().get_llm_fx_tools_list() + + def get_llm_fx_mapping(self): + + """Provides a current mapping of Tools to LLM Function Call - this mapping is used by LLMfx class to + orchestrate among multiple models deployed locally as tools. """ + + return _ModelRegistry().get_llm_fx_mapping() + + def get_test_script(self, model_name): + + """ Checks if a test script is available with the model repo - and if so, + retrieves the test set as a json dictionary """ + + test_set = None + + model_repo_path = LLMWareConfig().get_model_repo_path() + local_model_path = os.path.join(model_repo_path, model_name) + if os.path.exists(local_model_path): + model_files = os.listdir(local_model_path) + if "config.json" in model_files: + config_json = json.load(open(os.path.join(local_model_path, "config.json"), "r", + encoding="utf-8")) + if "test_set" in config_json: + test_set = config_json["test_set"] + + return test_set + + def tool_test_run(self, model_name, api_key=None, verbose=False, + # add more optional configurations to flow thru to the model inference + use_gpu=True, sample=True, get_logits=True, + max_output=100, temperature=-99, custom_test_script=None, + api_endpoint=None): + + """ Loads a tool, if required, and executes a series of test runs. Most of the input + parameters are optional configuration parameters that will be passed when the model is loaded + and instantiated. + + Note: only available for GGUF quantized 'tool' implementation models. """ + + model_card = self.lookup_model_card(model_name) + + agent_writer = AgentWriter() + + if not model_card: + raise ModelNotFoundException(model_name) + + model = self.load_model(model_name, api_key=api_key, use_gpu=use_gpu, sample=sample, + get_logits=get_logits,max_output=max_output, temperature=temperature, + api_endpoint=api_endpoint) + + if custom_test_script: + # custom_test_script can be any json file with list of json dictionary entries with + # keys corresponding to test set, e.g., "context", "query", "answer" + test_set = custom_test_script + else: + test_set = self.get_test_script(model_name) + + if test_set: + + if "function_call" not in model_card: + + # run traditional inference on test set + agent_writer.write(f"\nTest: {model_name}") + + for i, entries in enumerate(test_set): + + agent_writer.write(f"\nupdate: query - {i} - {entries['query']}") + + response = model.inference(entries["query"],add_context=entries["context"], + add_prompt_engineering="default_with_context") + + agent_writer.write(f"\nupdate: llm_response - {i} - {response['llm_response']}") + + if "answer" in entries: + agent_writer.write(f"update: gold answer - {i} - {entries['answer']}") + + else: + + agent_writer.write(f"\nTest: {model_name}") + + for i, entries in enumerate(test_set): + + text = entries["context"] + + # special case for nli + if "conclusion" in entries: + text = "Evidence: " + text + "\nConclusion: " + entries["conclusion"] + + # special case for boolean (question = params) + if "question" in entries: + params = entries["question"] + " (explain)" + response = model.function_call(text, params=[params]) + else: + # general case - use default params and function from model card + response = model.function_call(text) + + # if verbose: + agent_writer.write(f"\nupdate: context - test - {i} - {text}") + + agent_writer.write(f"update: 'llm_response' - test - {i} - {response['llm_response']}") + + logit_analysis = self.logit_analysis(response, model_card, model.hf_tokenizer_name, + api_key=api_key) + + if "ryg_string" in logit_analysis: + agent_writer.write(f"update: red-yellow-green confidence - {logit_analysis['ryg_string']}") + + if "confidence_score" in logit_analysis: + agent_writer.write(f"update: confidence score - {logit_analysis['confidence_score']}") + + if "marker_tokens" in logit_analysis: + if logit_analysis["marker_tokens"]: + agent_writer.write(f"update: marker tokens - {logit_analysis['marker_tokens']}") + + if "choices" in logit_analysis: + choices = logit_analysis["choices"] + if len(choices) > 0: + choices = choices[0] + + agent_writer.write(f"update: choices - {choices}") + + agent_writer.close() + + return 0 + + def list_function_call_models(self): + + """ Returns a list of model card dictionaries for models that implement function_calls.""" + + fc_model_list = [] + for models in self.global_model_list: + if "function_call" in models: + # confirm that value is positive + if models["function_call"]: + fc_model_list.append(models) + + return fc_model_list + + def logit_analysis(self, response, model_card, hf_tokenizer_name,api_key=None): + + """ Analyzes logits from llm response - currently exposed only as option for function + call inferences in HFGenerative and GGUFGenerative models. """ + + logit_analysis = [] + ryg_string = "" + vz_choices = [] + marker_token_probs = [] + low_confidence_choices = [] + confidence_score = -1 + + # only go ahead if logits found in response + if "logits" not in response: + logger.warning("ModelCatalog - logit_analysis requires a response dictionary with 'logits' key- skipping") + return logit_analysis + + try: + from colorama import Fore + red = Fore.RED + green = Fore.GREEN + yellow = Fore.YELLOW + color_reset = Fore.RESET + except: + logger.warning("ModelCatalog - logit analysis - could not import colorama - please import to see color coded" + "visualization of the output string confidence level.") + + # setting color inserts to empty + red = "" + green = "" + yellow = "" + color_reset = "" + + """ Analyzes logits from llm response """ + + # marker tokens for sentiment analysis + marker_tokens = [] + marker_token_lookup = {} + + if "marker_tokens" in model_card: + marker_tokens = model_card["marker_tokens"] + if "marker_token_lookup" in model_card: + marker_token_lookup = model_card["marker_token_lookup"] + + if "logits" in response: + + logits = response["logits"] + + # tokenizer load + if "tokenizer_local" in model_card: + tokenizer = LocalTokenizer(tokenizer_fn=model_card["tokenizer_local"]) + elif util.find_spec("transformers"): + # hf tokenizer name + pt_loader = PyTorchLoader(api_key=api_key, trust_remote_code=True, custom_loader=None) + tokenizer = pt_loader.get_tokenizer(hf_tokenizer_name) + else: + raise LLMWareException(message="Exception: could not identify tokenizer to use") + + try: + # pull bos attributes from tokenizer + # -- note: will be a list of .bos_id and .eos_id, e.g., [2], not 2 + bos_token_id = tokenizer.bos_id + bos_str = tokenizer.bos_token + + eos_token_id = tokenizer.eos_id + eos_str = tokenizer.eos_token + + if not isinstance(eos_token_id, list): + eos_token_id = [eos_token_id] + + if isinstance(bos_token_id, list): + if len(bos_token_id) > 0: + bos_token_id = bos_token_id[0] + else: + # set to llama as fallback + bos_token_id = 1 + except: + # unexpected - but if fail, then take llama defaults + bos_token_id = 1 + bos_str = "" + + eos_token_id = [2] + eos_str = "" + + ryg_string = "" + + token_probs = [] + marker_token_probs = [] + vz_choices = [] + vz_capture_on = False + + for i, toks in enumerate(response["output_tokens"]): + + # change - look directly for '[' in tokenized output + if "]" in tokenizer.decode(toks): + vz_capture_on = False + + if toks in marker_tokens: + + for x in range(0, len(logits[i])): + if logits[i][x][0] in marker_tokens: + + # new add 1020 - if from file, then dict number converted to str + if logits[i][x][0] in marker_token_lookup: + entry0 = marker_token_lookup[logits[i][x][0]] + + elif str(logits[i][x][0]) in marker_token_lookup: + entry0 = marker_token_lookup[str(logits[i][x][0])] + + else: + entry0 = "NA" + # end here + + new_entry = (entry0, + logits[i][x][0], + logits[i][x][1]) + marker_token_probs.append(new_entry) + + if vz_capture_on: + + new_entry = {} + for x in range(0,3): + key = "choice_" + str(x+1) + new_entry.update({key: [tokenizer.decode(logits[i][x][0]), + logits[i][x][1],logits[i][x][0]]}) + + # set confidence score as normalized logit value of first token in value zone + #TODO: need to assess whether averaging across multiple tokens more effective + + if len(vz_choices) == 0: + if logits[i][x][0] == toks: + confidence_score = logits[i][x][1] + + vz_choices.append(new_entry) + + # change - look for "[" directly in token decoded output + if "[" in tokenizer.decode(toks): + vz_capture_on = True + + # e.g., if toks in [2]: + if toks in eos_token_id: + break + + for x in range(0, len(logits[i])): + + if toks == logits[i][x][0]: + + token_probs.append(logits[i][x][1]) + + if logits[i][x][1] > 0.70: + ryg_string += green + tokenizer.decode([bos_token_id, logits[i][x][0]]) + + if 0.3 <= logits[i][x][1] <= 0.70: + ryg_string += yellow + tokenizer.decode([bos_token_id, logits[i][x][0]]) + + new_entry = {} + for y in range(0, 3): + key = "choice_" + str(y + 1) + new_entry.update({key: [tokenizer.decode(logits[i][y][0]), + logits[i][y][1], logits[i][y][0]]}) + + low_confidence_choices.append(new_entry) + + if logits[i][x][1] < 0.3: + ryg_string += red + tokenizer.decode([bos_token_id, logits[i][x][0]]) + + new_entry = {} + for y in range(0, 3): + key = "choice_" + str(y + 1) + new_entry.update({key: [tokenizer.decode(logits[i][y][0]), + logits[i][y][1], logits[i][y][0]]}) + + low_confidence_choices.append(new_entry) + + # removing hard-coded "" + ryg_string = ryg_string.replace(bos_str, "") + + logit_analysis = {"ryg_string": ryg_string + color_reset, "choices": vz_choices, + "marker_tokens": marker_token_probs, + "low_confidence_choices": low_confidence_choices, + "confidence_score": confidence_score} + + return logit_analysis + + def fc_output_values(self, model_name): + + """ Takes as input a model_name, and if the model is function-calling, then will output a list + of the expected function calling output values for the model. If no value provided, or no specific + expected 'constraints' on output values, then returns an empty list. """ + + output_values = [] + + model_card = self.lookup_model_card(model_name) + + if model_card: + if "fc_output_values" in model_card: + output_values = model_card["fc_output_values"] + + else: + logger.error(f"ModelCatalog - could not identify model card " + f"for selected model - {model_name} ") + + raise ModelNotFoundException(model_name) + + return output_values + + def fc_primary_keys(self, model_name): + + """ Takes as input a model_name, and if the model is function-calling, then will output a list of the + primary keys, if any, to be passed as parameters to the model. If no primary keys, then returns an + empty list. """ + + output_keys = [] + + model_card = self.lookup_model_card(model_name) + + if model_card: + if "primary_keys" in model_card: + output_keys = model_card["primary_keys"] + else: + logger.error(f"ModelCatalog - could not identify model card for " + f"selected model - {model_name}") + + raise ModelNotFoundException(model_name) + + return output_keys + + def remediate_function_call_string(self,input_string, dedupe_values=True): + + """ This method attempts to remediate a function call output string that can not be automatically + converted into a programmatic object. The method supports both DICT and LIST outputs. It is designed + to address the most common source of automatic failing, which is a premature termination at the end of the + string, usually due to a max_len cap, e.g., {'key': ['value1', value2', ..., 'val """ + + starter = 3 + keys = [] + values = [] + + # if very short output, then can not remediate - assume that a bigger problem happened with the inference + if len(input_string) < starter: + # llm response very short - could not remediate and convert to dict or list + return "string", input_string + + start = -1 + list_start = -1 + + # will scan the start of the string for either a dictionary start '{' or list start '[' + # if neither found, will return the original string + + for x in range(0, starter): + + if input_string[x] == "{": + # found dict starter + start = x + + if input_string[x] == "[": + # found list starter + list_start = x + + if start < 0 and list_start < 0: + # remediation not successful - could not find a start marker for dictionary or list + return "string", input_string + + # based on the start marker, determine the target output type + if start < 0 and list_start >= 0: + # try to build the string as a list output + list_type = True + key_or_value = "value" + response_type = "list" + start = list_start-1 + else: + # try to build the string as a dictionary output + list_type = False + key_or_value = "key" + response_type = "dict" + + string_on = False + key_tmp = "" + counter = 0 + output_dict = {} + output_list = [] + current_key = "" + + logger.debug(f"***test*** - remediation - input string - {input_string}") + + for y in range(start + 1, len(input_string)): + + # note: ASCII ORD conversion - 58 - ':' | 91 - '[' | 93 - ']' | 44 - ',' + + if string_on and ord(input_string[counter]) not in [34, 39]: + if ord(input_string[counter]) not in [91, 93, 58, 44]: + if ord(input_string[counter]) == 32 and not key_tmp.strip(): + pass + else: + key_tmp += input_string[counter] + + # edge case where there is quote around outer bracket + if ord(input_string[counter]) == 91 and string_on: + string_on = False + key_tmp = "" + + # string markers of ' and " + if ord(input_string[counter]) in [34, 39]: + + # insert new check if ' followed by 's' + exception_skip = False + if len(input_string) > counter+1: + if ord(input_string[counter+1]) in [115]: + exception_skip = True + # counter += 1 + # end - new check + + if not exception_skip: + + if not string_on: + string_on = True + key_tmp = "" + + else: + # end of string token + string_on = False + + if len(key_tmp) > 0: + + if not list_type: + if key_or_value == "key": + keys.append(key_tmp) + current_key = key_tmp + output_dict.update({current_key: []}) + + else: + values.append(key_tmp) + if current_key in output_dict: + output_dict[current_key].append(key_tmp) + else: + logger.warning("remediation - could not find key-value to correct - output " + "may be missing certain content in structured output.") + + key_tmp = "" + else: + output_list.append(key_tmp) + values.append(key_tmp) + key_tmp = "" + + if ord(input_string[counter]) == 58: + + if len(input_string) > counter + 5: + for z in range(1, 5): + if ord(input_string[counter + z]) == 91: + key_or_value = "value" + counter += z - 1 + break + + if ord(input_string[counter]) == 93: + key_or_value = "key" + + counter += 1 + if counter >= len(input_string): + break + + if not list_type: + # remediation successful in converting to dict output + if dedupe_values: + for keys, values in output_dict.items(): + output_dict[keys] = list(set(values)) + + return response_type, output_dict + else: + # remediation successful in converting to list output + if dedupe_values: + dd_output = [] + for elements in output_list: + if elements not in dd_output: + dd_output.append(elements) + + # not using set because it can change the order of the list from output + # output_list = list(set(output_list)) + + output_list = dd_output + + return response_type, output_list + + def analyze_sampling(self,response): + + """ Analyzes a llm response output dictionary and produces a 'sampling_stats' dictionary to provide + details on the effects, if any, of sampling in the output generation. """ + + sampling_stats = {} + + if "logits" not in response or "output_tokens" not in response: + logger.warning("ModelCatalog - function get_fx_scores requires a response dictionary with 'logits' key - " + "not found in the current response provided. Set the model parameters to 'get_logits=True'" + "for function call to provide logits") + return sampling_stats + + logits = response["logits"] + output_tokens = response["output_tokens"] + + not_top_selected = 0 + top_token_not_used = [] + + if len(output_tokens) == 0: + return sampling_stats + + for x in range(0, len(output_tokens)): + + top_selected = True + + if output_tokens[x] != logits[x][0][0] and x > 0: + top_selected = False + top_token_not_used.append((x, output_tokens[x], logits[x])) + + if not top_selected and x > 0: + not_top_selected += 1 + + tokens_considered = len(output_tokens) - 1 + if tokens_considered > 0: + percent_top_token = (tokens_considered - not_top_selected) / tokens_considered + else: + percent_top_token = 0.0 + + # sampling_stats added to the output dictionary + sampling_stats.update({"total_output_tokens": len(output_tokens), + "percent_top_token": round(percent_top_token, 3), + "not_top_tokens": top_token_not_used}) + + return sampling_stats + + def get_fx_scores(self,response, model_name, top_choices=3, logit_count=1, api_key=None): + + """ Provides useful metrics and scores derived from analyzing the logits and output tokens from function call + llm response - currently only supported for HFGenerative and GGUFGenerative models. + + Inputs: + -- llm response dictionary, including logits and output token + -- model_name which will be used to lookup the model card and get applicable tokenizer(s) + -- tokenizer will be used to decode output tokens, logits and identify key + 'value zone' markers for the output response, e.g., identify list boundaries '[' and ']' + -- top_choices - number of candidates to consider in each logit, e.g., top 3 choices considered + -- logit_count - number of tokens to consider in the value zone, whether the first only, or more + -- api_key - optional, if tokenizer in private repository requiring an api key + + Output (dictionary): + -- for each key in the output response, there is a list of the candidate logits in the value zone associated + with that key - the list will be the length of the logit count requested + -- a sampling_stats key will also be produced that will provide summary data on the number of 'value zone' + tokens, the percentage taken from the top output logit candidate and a list of the 'sampled', e.g., + 'not top' logits taken + """ + + # model name - look up model card + model_card = self.lookup_model_card(model_name) + + hf_tokenizer_name = None + tokenizer_local = None + + if "tokenizer" in model_card: + hf_tokenizer_name = model_card["tokenizer"] + + if "tokenizer_local" in model_card: + tokenizer_local = model_card["tokenizer_local"] + + # output is a dict of dict + output = {} + + if "logits" not in response or "output_tokens" not in response: + logger.warning("ModelCatalog - function get_fx_scores requires a response dictionary with 'logits' key - " + "not found in the current response provided. Set the model parameters to 'get_logits=True'" + "for function call to provide logits") + return output + + logits = response["logits"] + + keys_list = [] + llm_response = response["llm_response"] + + if isinstance(llm_response, dict): + for key, value in llm_response.items(): + keys_list.append(key) + elif isinstance(llm_response, list): + keys_list.append("llm_response") + else: + keys_list.append("llm_response") + + # tokenizer load + if tokenizer_local: + tokenizer = LocalTokenizer(tokenizer_fn=model_card["tokenizer_local"]) + elif hf_tokenizer_name and util.find_spec("transformers"): + # hf tokenizer name + pt_loader = PyTorchLoader(api_key=api_key, trust_remote_code=True, custom_loader=None) + tokenizer = pt_loader.get_tokenizer(hf_tokenizer_name) + else: + raise LLMWareException(message="Exception: could not identify tokenizer to use") + + vz_choices = [] + vz_capture_on = False + key_counter = 0 + + min_threshold = 0.005 + vz_logits = 0 + vz_top_logits = 0 + top_token_not_used = [] + + for i, toks in enumerate(response["output_tokens"]): + + decoded = tokenizer.decode(toks) + + if "]" in decoded: + vz_capture_on = False + if vz_choices: + output.update({keys_list[key_counter]: vz_choices}) + key_counter += 1 + vz_choices = [] + + if vz_capture_on: + + new_entry = {} + if toks == logits[i][0][0]: + vz_top_logits += 1 + else: + # the output token does not correspond to the logit with the highest score, so there was a + # 'sampling' effect to this generation - adding this token and corresponding logit to be saved + # and provided as output in 'sampling_stats' + top_token_not_used.append((i, toks, logits[i])) + + vz_logits += 1 + + for x in range(0, top_choices): + + if logits[i][x][1] >= min_threshold: + new_entry.update({tokenizer.decode(logits[i][x][0]): round(logits[i][x][1], 3)}) + + if len(vz_choices) < logit_count: + vz_choices.append(new_entry) + + if "[" in decoded: + vz_capture_on = True + vz_choices = [] + + if vz_top_logits > 0: + top_token_in_value_zone = round(vz_logits / vz_top_logits, 2) + else: + top_token_in_value_zone = 0.0 + + # sampling_stats added to the output dictionary + output.update({"sampling_stats": {"total_vz_tokens": vz_logits, + "percent_top_token": top_token_in_value_zone, + "not_top_tokens": top_token_not_used} + }) + + return output + + def gpu_available(self, suppress_warnings=True, driver_min_levels=None): + + """ Checks if CUDA GPU drivers found on machine, and whether the drivers are at + the required minimum level. + + -- driver_min_level is a tuple of integers consisting of the major/minor driver level, e.g., (525, 15) + -- if no driver_min_level is passed, then the test will be skipped and come back False by default. + """ + + major_driver = 0 + minor_driver = 0 + + result = {"gpu_found": False, "drivers_current": False, + "gpu_name": "", "driver": "", "multiple_gpu": False} + + + try: + from subprocess import Popen, PIPE + except: + if not suppress_warnings: + logger.warning("ModelCatalog - check gpu availability - unable to check if gpu available") + return result + + if sys.platform.lower() == "win32": + nvidia_smi = shutil.which('nvidia-smi') + elif sys.platform.lower().startswith("linux"): + nvidia_smi = "nvidia-smi" + else: + if not suppress_warnings: + logger.warning("ModelCatalog - check gpu availability - only check for CUDA drivers on Windows or Linux") + return result + + try: + gpu_pipe = Popen([nvidia_smi, "--query-gpu=index,driver_version,name","--format=csv,noheader,nounits"], + stdout=PIPE) + gpu, errors = gpu_pipe.communicate() + except Exception as e: + gpu = [] + errors = e + + if gpu: + + result["gpu_found"] = True + + # only looking at 'first' gpu + results = str(gpu).split(",") + if len(results) > 1: + + #TODO: handle multiple GPUs on device! + driver_index = results[0].strip().encode('utf') + + driver_level = results[1].strip() + result["driver"] = driver_level + + if len(results) > 2: + result["gpu_name"] = results[2].strip() + + if driver_min_levels: + + driver_split = driver_level.split(".") + + if len(driver_split) > 0: + try: + major_driver = int(driver_split[0].strip()) + if len(driver_split) > 1: + minor_driver = int(driver_split[1].strip()) + except: + pass + + if major_driver > driver_min_levels[0] or (major_driver == driver_min_levels[0] + and minor_driver >= driver_min_levels[1]): + result["drivers_current"] = True + + else: + result["drivers_current"] = False + logger.warning(f"ModelCatalog - check gpu availability - CUDA device found - but drivers " + f"look out of date, relative to required min levels: \n" + f"--drivers found: {driver_level}\n" + f"--min required: {driver_min_levels}\n") + + return result + + +class PromptCatalog: + + """ PromptCatalog manages prompt styles and prompt wrappers and builds prompt templates for inference + generation. """ + + def __init__(self): + + self.prompt_catalog = global_default_prompt_catalog + self.prompt_wrappers = _ModelRegistry().prompt_wrappers + self.prompt_wrapper_lookup = _ModelRegistry().get_wrapper_list() + + self.prompt_list = self.list_all_prompts() + + def lookup_prompt(self, prompt_name): + + """ Looks up a predefined prompt template by prompt_name. """ + + for prompts in self.prompt_catalog: + if prompts["prompt_name"] == prompt_name: + return prompts + + return None + + def get_all_prompts(self): + + """ Returns all predefined prompts. """ + + return self.prompt_catalog + + def list_all_prompts(self): + + """ Returns a list of all predefined prompts. """ + + prompt_list = [] + for prompt in self.prompt_catalog: + if "prompt_name" in prompt: + prompt_list.append(prompt["prompt_name"]) + return prompt_list + + def parse_instruction_for_user_vars(self, prompt_card, inference_dict=None): + + """ Utility method that looks for user_vars in prompt card to dynamically insert into Prompt. """ + + # if no user vars key in prompt_card, then return instruction unchanged + + if "user_vars" not in prompt_card: + return prompt_card["instruction"] + + if not prompt_card["user_vars"]: + return prompt_card["instruction"] + + # if no inference_dict, then define as empty dictionary + if not inference_dict: + inference_dict = {} + + # in this case, will 'parameterize' and dynamically update instruction + tokens = prompt_card["instruction"].split(" ") + updated_instruction = "" + + for i, t in enumerate(tokens): + + if t.startswith("{{") and t.endswith("}}"): + + t_core = t[2:-2] + + # if value found for key in the inference dict, then apply as true 'user_vars' + if t_core in inference_dict: + new_inserted_token = inference_dict[t_core] + updated_instruction += str(new_inserted_token) + " " + else: + # apply default value found in the prompt card as back-up + if t_core in prompt_card["user_vars"]: + new_inserted_token = prompt_card["user_vars"][t_core] + updated_instruction += str(new_inserted_token) + " " + + else: + updated_instruction += t + " " + + logger.debug(f"PromptCatalog - constructed dynamic instruction - {updated_instruction}") + + return updated_instruction.strip() + + def build_core_prompt(self, prompt_card=None, prompt_name=None, separator="\n", query=None, context=None, + inference_dict=None): + + """ Builds the core prompt from the prompt_card template. """ + + if not context: context = "" + if not query: query = "" + + if not prompt_card and not prompt_name: + # error - returning query + logger.warning("PromptCatalog - no prompt selected in PromptCatalog().build_core_prompt") + prompt_dict = {"core_prompt": context + "\n" + query, "prompt_card": {}} + return prompt_dict + + if not prompt_card: + prompt_card = PromptCatalog().lookup_prompt(prompt_name) + + logger.debug(f"PromptCatalog - prompt_card - {prompt_card}") + + core_prompt = "" + + if prompt_card: + for keys in prompt_card["run_order"]: + + if keys == "instruction": + # special handler + instruction = self.parse_instruction_for_user_vars(prompt_card, inference_dict=inference_dict) + core_prompt += instruction + separator + else: + if not keys.startswith("$"): + core_prompt += prompt_card[keys] + separator + else: + if keys == "$query": + core_prompt += query + separator + if keys == "$context": + core_prompt += context + separator + + # update instruction, if user_vars accepted in instruction + + """ + if "instruction" in prompt_card: + prompt_card["instruction"] = self.parse_instruction_for_user_vars(prompt_card,inference_dict=inference_dict) + core_prompt += prompt_card["instruction"] + """ + + prompt_dict = {"core_prompt": core_prompt, "prompt_card": prompt_card} + + logger.debug(f"PromptCatalog - prompt created - {prompt_dict}") + + return prompt_dict + + def add_custom_prompt_card(self, prompt_name, run_order_list, prompt_dict, prompt_description=None): + + """ Registers a new custom prompt_card with 'run_order_list' that shows how to assemble the components + of a Prompt. """ + + new_prompt_card = {"prompt_name": prompt_name, + "prompt_description": prompt_description, + "run_order": run_order_list} + + for keys, values in prompt_dict.items(): + new_prompt_card.update({keys: values}) + + self.prompt_catalog.append(new_prompt_card) + + return new_prompt_card + + def apply_prompt_wrapper(self, text, prompt_wrapper, + separator="\n", + instruction=None, + chat_history=None): + + """ Applies the selected prompt_wrapper to the prompt. """ + + output_text = text + + if prompt_wrapper not in self.prompt_wrappers: + logger.info(f"PromptCatalog - apply_prompt_wrapper - selected wrapper - {prompt_wrapper} - could not be identified - " + f"returning text prompt without any special format wrapping") + + return output_text + + if prompt_wrapper == "chatgpt": + return self.wrap_chatgpt_sample(text, instruction) + + else: + wrapped_prompt = self.wrap_custom(text, prompt_wrapper, + instruction=instruction, + chat_history=chat_history) + + return wrapped_prompt + + def wrap_custom(self, text, wrapper_type, chat_history=None, + instruction=None): + + """ Provides option for chat history, packaged as a list of 'turns' + with each turn consisting of two dictionary entries - + 'user' and 'assistant' """ + + #TODO: apply safeguards to max output + + prompt_out = "" + + if wrapper_type in self.prompt_wrapper_lookup: + + prompt_template = self.prompt_wrapper_lookup[wrapper_type] + + if "system_start" in prompt_template: + + if prompt_template["system_start"] != "": + + prompt_out += prompt_template["system_start"] + + if instruction: + prompt_out += instruction + else: + prompt_out += "You are a helpful assistant." + + if "system_stop" in prompt_template: + prompt_out += prompt_template["system_stop"] + + if chat_history: + + for turn in chat_history: + + # user part of turn + if "main_start" in prompt_template: + prompt_out += prompt_template["main_start"] + + prompt_out += turn["user"] + + if "main_stop" in prompt_template: + prompt_out += prompt_template["main_stop"] + + # assistant part of turn + if "start_llm_response" in prompt_template: + prompt_out += prompt_template["start_llm_response"] + + prompt_out += turn["assistant"] + + if "main_start" in prompt_template: + + prompt_out += prompt_template["main_start"] + prompt_out += text + + if "main_stop" in prompt_template: + prompt_out += prompt_template["main_stop"] + + if "start_llm_response" in prompt_template: + prompt_out += prompt_template["start_llm_response"] + + else: + prompt_out = text + + return prompt_out + + def wrap_chatgpt_sample(self, text, instruction): + + """ Applies chatgpt format wrapper to a prompt. """ + + if not instruction: + instruction = "You are a helpful assistant." + + new_sample = [{"role": "system", "content": instruction}, + {"role": "user", "content": text}] + + return new_sample + + +class InferenceHistory: + + """ Global State History of All Inferences Completed in Session """ + + base_model_keys = ["llm_response", "usage", "logits", "output_tokens", "prompt", "add_context","final_prompt", + "model_name", "model_card", "temperature", "add_prompt_engineering", + "model_class", "model_category", "prompt_wrapper", "time_stamp" + ] + + inference_history = [] + + global_inference_counter = 0 + + save = True + + @classmethod + def get_base_model_keys(cls): + return cls.base_model_keys + + @classmethod + def add_base_model_key(cls, new_key): + if new_key not in cls.base_model_keys: + cls.base_model_keys.append(new_key) + return True + + @classmethod + def del_base_model_key(cls, key_to_delete): + if key_to_delete in cls.base_model_keys: + del cls.base_model_keys[key_to_delete] + return True + + @classmethod + def get_transactions(cls): + """ List current view of implemented supported vector db for embeddings. """ + return cls.inference_history + + @classmethod + def add_transaction(cls, model_state_dict): + """ Adds a vector db including the module and class. """ + cls.inference_history.append(model_state_dict) + return True + + @classmethod + def get_global_inference_count(cls): + return cls.global_inference_counter + + @classmethod + def increment_global_inference_count(cls): + cls.global_inference_counter += 1 + return cls.global_inference_counter + + @classmethod + def reset_global_inference_count(cls): + cls.global_inference_counter = 0 + return cls.global_inference_counter + + @classmethod + def get_save_status(cls): + return cls.save + + @classmethod + def set_save_status(cls, status): + if isinstance(status, bool): + cls.save = status + else: + raise LLMWareException(message="Exception: save status must be boolean - True/False") + + +def register(kv_dict): + + """ Default register function called after each Model inference activity. This method can be over-ridden and + customized by re-routing the LLMWareConfig as follows: + + `LLMWareConfig().set_config('model_register', {'module': 'my_module', 'class': 'my_register_fx'}) + + `module` currently points to this module: 'llmware.models' + `class` currently points to this method: 'register' + """ + + # if save status set to False, then skip + if not InferenceHistory().get_save_status(): + logger.debug(f"InferenceHistory - skipping registration since save status is False") + return True + + for k, v in kv_dict.items(): + logger.debug(f"InferenceHistory - register: {k} - {v}") + + InferenceHistory().increment_global_inference_count() + + logger.debug(f"InferenceHistory - global inference counter - {InferenceHistory().get_global_inference_count()}") + + # by default, will register all generative inferences, but takes no action to track embedding inferences + if "model_category" in kv_dict: + if kv_dict["model_category"] == "generative": + InferenceHistory().add_transaction(kv_dict) + + return True + + +def post_init(kv_dict): + + """ Not implemented by default. """ + logger.debug(f"Model Load - in post_init - not implemented - returning True - no action taken") + + return True + + +def validate(kv_dict): + + """ Not implemented by default. """ + logger.debug(f"Model Load - validate - not implemented - returning True - no action taken") + + return True + + +def preview(kv_dict): + + """ Not implemented by default. """ + logger.debug(f"Model Load - preview - not implemented - returning True - no action taken") + + return True + + +def route_optimizer(kv_dict): + + """ Not implemented by default. """ + logger.debug(f"Model Route Optimizer - not implemented - returning True - no action taken") + + return True + + +class BaseModel: + + """ BaseModel class subclassed by all models. Should not be instantiated directly. Provides several + common utility methods across each of the Model class implementations. """ + + def __init__(self, **kwargs): + + # InferenceHistory provides a set of state parameters to be captured from each Model instantiation + self.base_model_keys = InferenceHistory().get_base_model_keys() + + self.time_stamp = None + self.model_class = None + self.model_category = None + self.model_card = {} + + self.tokenizer = None + + self.URL_BASE = None + self.api_endpoint = None + self.unlock_on_completion = None + + # parameters moved to base model + self.separator = "\n" + self.instruction_following = False + self.prompt_wrapper = None + self.add_prompt_engineering = True + + # output inference parameters + for keys in self.base_model_keys: + if keys in kwargs: + setattr(self, keys, kwargs[keys]) + else: + setattr(self, keys, None) + + def to_state_dict(self): + + """ Writes selected model state parameters to dictionary. """ + + state_dict = {} + for keys in self.base_model_keys: + if hasattr(self, keys): + state_dict.update({keys: getattr(self, keys)}) + + return state_dict + + def load_model_for_inference(self, loading_instructions): + # not implemented in base model + pass + + def method_resolver(self, config_name): + + """ Resolves method to invoke selected function. """ + + process_class = "" + process_method = "" + + method_exec = None + + state_dict = self.to_state_dict() + process = LLMWareConfig().get_config(config_name) + process_module = process["module"] + + if "class" in process: + process_class = process["class"] + + if "method" in process: + process_method = process["method"] + + module_exec = importlib.import_module(process_module) + + if process_class: + if hasattr(module_exec, process_class): + class_exec = getattr(module_exec, process_class)() + + if process_method: + if hasattr(class_exec, process_method): + method_exec = getattr(class_exec, process_method) + else: + if hasattr(module_exec, process_method): + method_exec = getattr(module_exec, process_method) + + if method_exec: + + success = method_exec(state_dict) + + if isinstance(success, dict): + # write attributes, if any, to the Model instance state + for k, v in success.items(): + setattr(self, k, v) + + return True + + def set_api_key(self, api_key, env_var="USER_MANAGED_API_KEY"): + + """ Sets the API key - generally not needed for self-hosted models. """ + + os.environ[env_var] = api_key + logger.info("BaseModel - added and stored api_key in environmental " + "variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_API_KEY"): + + """ Gets API key from os.environ variable. """ + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("BaseModel - _get_api_key could not successfully " + "retrieve value from: %s ", env_var) + + return self.api_key + + def post_init(self): + return self.method_resolver("model_post_init") + + def register(self): + + if self.unlock_on_completion: + ModelResources().unlock(self.unlock_on_completion) + + return self.method_resolver("model_register") + + def validate(self): + return self.method_resolver("model_validate") + + def preview(self): + return self.method_resolver("model_preview") + + def _lookup_endpoint(self, api_name, api_catalog): + + """ Internal lookup utility to pull api card. """ + + for entries in api_catalog: + if entries["api_name"] == api_name: + return entries + + return {} + + def prune_context(self, ctx, front=100,back=100): + + # apply pruning of stop words + pruned_ctx = Utilities().prune_stop_words(ctx,front=front,back=back) + + # test len + pruned_tokens = self.count_tokens(pruned_ctx) + + logger.info(f"BaseModel - prune_context - token count - {pruned_tokens}") + + # extra pruning for very large contexts + # need to reduce for 14B parameter models + if pruned_tokens > 16000: + start = pruned_ctx[0:1000] + end = pruned_ctx[pruned_tokens-5000:] + super_pruned = start + end + pruned_tokens = self.count_tokens(super_pruned) + logger.info(f"BaseModel - prune_context - token count - {pruned_tokens}") + pruned_ctx = super_pruned + + return pruned_ctx + + def count_tokens(self, ctx, tokenizer=None): + + if not tokenizer: + tokenizer = self.tokenizer + + toks = tokenizer.encode(ctx) + tok_len = len(toks.ids) + return tok_len + + def prompt_engineer(self, query, context, inference_dict): + + """ Applies prompt and templating preparation. """ + + # adding chat history to inference_dict handler + + chat_history = None + system_instruction = None + + if inference_dict: + if "system_instruction" in inference_dict: + system_instruction = inference_dict["system_instruction"] + + if "chat_history" in inference_dict: + chat_history = inference_dict["chat_history"] + + if self.instruction_following: + logger.info(f"BaseModel - prompt_engineer - found deprecated setting - " + f"instruction_following set to True - may cause unpredictable results.") + + # self.instruction_following = False + + # if loaded model was not pretrained on instruction_following, then skip any instructions + if not self.instruction_following: + + if context: + output = context + "\n" + query + else: + output = query + + # unlikely that there would be an 'instruct wrapping' on text, but allow for possibility + if self.prompt_wrapper: + output = PromptCatalog().apply_prompt_wrapper(output, + self.prompt_wrapper, + chat_history=chat_history, + instruction=system_instruction) + + return output + + # move ahead to add instructions and prompt engineering + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, + context=context, + inference_dict=inference_dict) + + if prompt_dict: + prompt_engineered = prompt_dict["core_prompt"] + else: + # default case + prompt_engineered = "Please read the following text: " + context + self.separator + prompt_engineered += "Based on this text, please answer the question: " + query + self.separator + prompt_engineered += "Please answer the question only with facts provided in the materials. " \ + "If the question can not be answered in the materials, then please " \ + "respond 'Not Found.'" + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper, + instruction=None) + + return prompt_engineered + + def function_call(self, context, function=None, params=None, get_logits=False, + temperature=-99, max_output=None): + + """ This is the key inference method for SLIM models - takes a context passage and a key list + which is packaged in the prompt as the keys for the dictionary output""" + + output_response = {} + + return output_response + + def fc_prompt_engineer(self, context, params=None, function=None, + trailing_space= ""): + + """ Prompt engineering for Function Call prompts. """ + + # prepare SLIM prompt + class_str = "" + for key in params: + class_str += str(key) + ", " + if class_str.endswith(", "): + class_str = class_str[:-2] + + f = str(function) + + # key templating format for SLIM function calls + full_prompt = ": " + context + "\n" + "<{}> {} ".format(f, class_str, f) + "\n:" + + full_prompt = full_prompt + trailing_space + + return full_prompt + + def close(self): + + """ General purpose 'close' method with any special wind-down + procedures at the time of closing out an inferencing session. """ + + pass + + +class ONNXGenerativeModel(BaseModel): + + """ONNXGenerativeModel class implements the ONNX Runtime generative model interface, and is used generally for + models converted from Pytorch into ONNX for faster inference performance and packaging on Windows platforms + and x86 architectures. """ + + def __init__(self, model_name=None, api_key=None, model_card=None, instruction_following=False, context_window=2048, + sample=True, max_output=100, temperature=0.3, get_logits=False, api_endpoint=None, **kwargs): + + super().__init__() + + self.model_class = "ONNXGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + self.model_name = model_name + self.hf_tokenizer_name = model_name + self.model = None + self.tokenizer = None + self.generator = None + self.context_window = context_window + self.sample = sample + self.get_logits = get_logits + self.auto_remediate_function_call_output = True + + # Function Call parameters + self.model_card = model_card + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.primary_keys = None + self.function = None + self.fc_supported = False + self.tool_type = None + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + if "context_window" in model_card: + self.context_window = model_card["context_window"] + + # insert dynamic onnx load here + if not api_endpoint: + + global GLOBAL_ONNX_GENAI_RUNTIME + + if not GLOBAL_ONNX_GENAI_RUNTIME: + + if util.find_spec("onnxruntime_genai"): + + try: + global og + og = importlib.import_module("onnxruntime_genai") + GLOBAL_ONNX_GENAI_RUNTIME = True + except: + raise LLMWareException(message="ONNXGenerativeModel: could not load onnxruntime_genai module. " + "If you have pip installed the library, then please check " + "that your platform is supported by onnxruntime.") + + else: + import platform + if platform.system() == "Darwin": + raise LLMWareException(message=f"ONNXGenerativeModel: identified current platform as 'Mac OS' " + f"which is not supported for onnxruntime_genai currently. " + f"\nWe would recommend using GGUF for generative inference on a " + f"Mac, or if you wish to use ONNXGenerativeModel, then please " + f"shift to a supported Windows or Linux platform.") + + raise LLMWareException(message="ONNXGenerativeModel: need to import " + "onnxruntime_genai to use this class, e.g., 'pip3 install " + "onnxruntime_genai`") + + # end dynamic import here + + if model_name and not api_endpoint: + + if not self.model_card: + self.model_card = ModelCatalog().lookup_model_card(self.model_name) + + if self.model_card: + if "hf_repo" in self.model_card: + hf_repo_name = self.model_card["hf_repo"] + self.hf_tokenizer_name = hf_repo_name + + self.model = None + self.tokenizer = None + self.tokenizer_stream = None + + # this can be over-ridden post initiation if needed for custom models + self.prompt_wrapper = "human_bot" + self.instruction_following = False + + self.params = None + + # set specific parameters associated with custom models + # note - these two parameters will control how prompts are handled - model-specific + self.prompt_wrapper = "human_bot" + self.instruction_following = instruction_following + + if not model_card: + # safety - empty iterable rather than 'None' + model_card = {} + + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + else: + self.instruction_following = False + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + # sets trailing space default when constructing the prompt + # in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves + # performance + + self.trailing_space = "" + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + + self.model_type = None + self.config = None + + # parameters on context len + output generation + self.max_total_len = self.context_window + self.max_input_len = int(0.5 * self.context_window) + self.llm_max_output_len = int(0.5 * self.context_window) + + # key output parameters + self.max_output = max_output + self.target_requested_output_tokens = self.max_output + + self.model_architecture = None + self.separator = "\n" + + # use 0 as eos token id by default in generation -> but try to pull from model config + self.eos_token_id = 0 + + # will load model and inference onto gpu, + # if (a) CUDA available and (b) use_gpu_if_available set to True (default) + # TODO: CUDA option handling for ONNX models + if not api_endpoint: + self.use_gpu = False + else: + self.use_gpu = False + + # no api key expected or required + self.api_key = api_key + + self.error_message = "\nUnable to identify and load ONNX model." + + # temperature settings + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at default of 0.3 + self.temperature = 0.3 + + self.add_prompt_engineering = False + self.add_context = "" + self.context = "" + self.prompt = "" + + self.api_endpoint = api_endpoint + + self.model_repo_path = None + + self.post_init() + + def load_model_for_inference(self, loading_directions, model_card=None): + + """ Loads ONNX Model from local path using loading directions. """ + + global og + + self.model_repo_path = loading_directions + + if model_card: + self.model_card = model_card + + self.validate() + + onnx_model_path = os.path.join(LLMWareConfig().get_model_repo_path(), + self.model_name) + + try: + self.model = og.Model(onnx_model_path) + self.tokenizer = og.Tokenizer(self.model) + self.tokenizer_stream = self.tokenizer.create_stream() + except: + raise LLMWareException(message=f"ONNXGenerativeModel - unable to load and instantiate the model at: " + f"\n{onnx_model_path}\nThis could be for a number of reasons, but " + f"most likely is one of the following:" + f"\n1. onnxruntime not installed correctly." + f"\n2. platform (e.g, Mac) is not supported by current ONNX Build." + f"\n3. model could not be found at this path, or is not a valid ONNX model." + ) + + search_options = {} + + # max length set at minimum of 2048 + # adjusted to the actual model context window (if available) + # currently cap at 'safety' max of 8192 + # --seems to have performance impact at larger lengths + + max_length = max(2048, self.max_total_len) + if max_length > 8192: + max_length = 8192 + + search_options['max_length'] = max_length + + self.params = og.GeneratorParams(self.model) + self.params.set_search_options(**search_options) + + return self + + def unload_model(self): + + """ Remove model pointer from memory space. In most use cases, simply deleting the model pointer will suffice + to trigger Python memory cleanup with an explicit call to gc.collect(). This is WIP and will continue + to test different scenarios to explore the best 'safe' unload steps. """ + + self.model = None + self.tokenizer = None + import gc + gc.collect() + + return True + + def set_api_key(self, api_key, env_var="USER_MANAGED_ONNX_API_KEY"): + + """ Sets the API key - generally not needed for ONNX self-hosted models. """ + + os.environ[env_var] = api_key + logger.info("ONNXGenerativeModel - added and stored ONNX api_key in " + "environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_ONNX_API_KEY"): + + """ Gets API key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("ONNXGenerativeModel - _get_api_key could not successfully " + "retrieve value from: %s ", env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Not Used for ONNXGenerativeModel class - Quick approximate token counter - + uses default tokenizer so may have minor differences from the model's actual tokenization. """ + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + + return len(toks) + + def prompt_engineer(self, query, context, inference_dict): + + """ Applies prompt and templating preparation. """ + + # if loaded model was not pretrained to require instruction_following, then skip any instructions + if not self.instruction_following: + + if context: + output = context + "\n" + query + else: + output = query + + # unlikely that there would be an 'instruct wrapping' on text, but allow for possibility + if self.prompt_wrapper: + output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper, + instruction=None) + + return output + + # move ahead to add instructions and prompt engineering + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, + context=context, + inference_dict=inference_dict) + + if prompt_dict: + prompt_engineered = prompt_dict["core_prompt"] + else: + # default case + prompt_engineered = "Please read the following text: " + context + self.separator + prompt_engineered += "Based on this text, please answer the question: " + query + self.separator + prompt_engineered += "Please answer the question only with facts provided in the materials. " \ + "If the question can not be answered in the materials, then please " \ + "respond 'Not Found.'" + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper, + instruction=None) + + return prompt_engineered + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None): + + """ Executes generation inference on model. """ + + from llmware.configs import ONNXConfig + + legacy = ONNXConfig().get_legacy_flag() + + global og + + # first prepare the prompt + t0 = time.time() + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + # show warning if function calling model + if self.fc_supported: + logger.warning("ONNXGenerativeModel - this is a function calling model - using .inference may lead to " + "unexpected results. Recommended to use the .function_call method to ensure correct prompt " + "template packaging.") + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + # START - route to api endpoint + if self.api_endpoint: + return self.inference_over_api_endpoint(self.prompt, context=self.add_context, + inference_dict=inference_dict) + # END - route to api endpoint + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # text_prompt = prompt_final + "\n" + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + text_prompt = prompt_final + self.trailing_space + + input_tokens = self.tokenizer.encode(text_prompt) + + if legacy: + self.params.input_ids = input_tokens + + token_count = 0 + output = "" + + try: + generator = og.Generator(self.model, self.params) + except: + raise LLMWareException(message=f"ONNXGenerativeModel - attempt to instantiate ONNX generator with " + f"model and prompt failed. This is most likely due to an error in the " + f"installation of the onnxruntime, or a problem with loading either the " + f"model or the input tokens.") + + # borrow 'get_first_token_speed' config from GGUFConfigs + get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed") + t_gen_start = time.time() + first_token_processing_time = -1.0 + + if not legacy: + generator.append_tokens(input_tokens) + + while not generator.is_done(): + + token_count += 1 + + if legacy: + generator.compute_logits() + + generator.generate_next_token() + + # get logits - in most cases, get_logits is set to False for basic inference + + if self.get_logits: + logit = generator.get_output("logits") + self.register_top_logits(logit) + + new_token = generator.get_next_tokens()[0] + + # first token capture + if get_first_token_speed: + if token_count == 1: + first_token_processing_time = time.time() - t_gen_start + # first token capture ends here + + if self.get_logits: + self.output_tokens.append(new_token) + + output += self.tokenizer_stream.decode(new_token) + + # add stream on/off options + # print(self.tokenizer_stream.decode(new_token), end="", flush=True) + + if token_count > self.max_output: + break + + # direct deletion of generator recommended in onnxruntime_genai examples + del generator + + llm_response = {"llm_response": output, "usage": {}} + + usage = {"input": len(input_tokens), + "output": token_count, + "total": len(input_tokens) + token_count, + "metric": "tokens", + "processing_time": time.time() - t0} + + if get_first_token_speed: + usage.update({"first_token_processing_time": first_token_processing_time}) + + output_response = {"llm_response": output, "usage": usage} + + if self.get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def fc_prompt_engineer(self, context, params=None, function=None): + + """ Prompt engineering for Function Call prompts. """ + + if not params: + params = self.primary_keys + + if not function: + function = self.function[0] + + # prepare SLIM prompt + class_str = "" + for key in params: + class_str += str(key) + ", " + if class_str.endswith(", "): + class_str = class_str[:-2] + + f = str(function) + + # key templating format for SLIM function calls + full_prompt = ": " + context + "\n" + "<{}> {} ".format(f, class_str, f) + "\n:" + + full_prompt = full_prompt + self.trailing_space + + return full_prompt + + def register_top_logits(self, logit): + + """ Gets the top logits and keeps a running log for output analysis. """ + + # logit will be in form of (1,1,vocab_len), for all but the first logit + # if first logit (will have shape of context len - add [-1]) + + if logit.shape[1] > 1: + # used for first logit with shape, e.g., (1,input_token_len,vocab_size) + logit_array = logit.squeeze()[-1] + else: + # all other logits after the first token + logit_array = logit.squeeze() + + logit_size = logit.shape[-1] + + # useful check on shape of logit_array + logit_array_size = logit_array.shape + + sm = np.exp(logit_array) / sum(np.exp(logit_array)) + + sm_sorted = np.sort(sm) + sm_args_sorted = np.argsort(sm) + + top_logits = [] + + for x in range(0, self.top_logit_count): + # round the float number to 3 digits + pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3)) + top_logits.append(pair) + + self.logits_record.append(top_logits) + + return top_logits + + def function_call(self, context, function=None, params=None, get_logits=True, + temperature=-99, max_output=None): + + """ This is the key inference method for SLIM models - takes a context passage and a key list + which is packaged in the prompt as the keys for the dictionary output""" + + from llmware.configs import ONNXConfig + legacy = ONNXConfig().get_legacy_flag() + + t0 = time.time() + + self.context = context + + if not self.fc_supported: + logger.warning(f"ONNXGenerativeModel - loaded model does not support function calls. " + "Please either use the standard .inference method with this model, or use a " + "model that has 'function_calls' key set to True in its model card.") + return [] + + # reset and start from scratch with new function call + self.output_tokens = [] + self.logits_record = [] + + if temperature != -99: + self.temperature = temperature + + if max_output: + self.target_requested_output_tokens = max_output + + if get_logits: + self.get_logits = get_logits + + if params: + self.primary_keys = params + + if function: + self.function = function + + if not self.primary_keys: + logger.warning(f"ONNXGenerativeModel - function call - no keys provided - " + f"function call may yield unpredictable results") + + self.preview() + + # START - route to api endpoint + + if self.api_endpoint: + return self.function_call_over_api_endpoint(model_name=self.model_name, + context=self.context, params=self.primary_keys, + function=self.function, + api_key=self.api_key, get_logits=self.get_logits) + + # END - route to api endpoint + + prompt = self.fc_prompt_engineer(self.context, params=self.primary_keys, function=self.function) + + input_tokens = self.tokenizer.encode(prompt) + + if legacy: + self.params.input_ids = input_tokens + + token_count = 0 + output = "" + + try: + generator = og.Generator(self.model, self.params) + except: + raise LLMWareException(message=f"ONNXGenerativeModel - attempt to instantiate ONNX generator with " + f"model and prompt failed. This is most likely due to an error in the " + f"installation of the onnxruntime, or a problem with loading either the " + f"model or the input tokens.") + + if not legacy: + generator.append_tokens(input_tokens) + + while not generator.is_done(): + + token_count += 1 + + if legacy: + generator.compute_logits() + + # to get logit value + if self.get_logits: + logit = generator.get_output("logits") + self.register_top_logits(logit) + + generator.generate_next_token() + + new_token = generator.get_next_tokens()[0] + + if self.get_logits: + self.output_tokens.append(new_token) + + output += self.tokenizer_stream.decode(new_token) + + # add as streaming option to turn on/off + # print(self.tokenizer_stream.decode(new_token), end="", flush=True) + + if token_count >= self.max_output: + break + + # done with generator + del generator + + llm_response = {"llm_response": output, "usage": {}} + + usage = {"input": len(input_tokens), + "output": token_count, + "total": len(input_tokens) + token_count, + "metric": "tokens", + "processing_time": time.time() - t0} + + output_response = {"llm_response": output, "usage": usage} + + # end - post-processing + + try: + import ast + output_value = ast.literal_eval(output) + + output_type = "dict" + + # allow for multiple valid object types - will expand over time + if isinstance(output_value, dict): output_type = "dict" + if isinstance(output_value, list): output_type = "list" + + usage.update({"type": output_type}) + + except: + # could not convert automatically to python object + output_type = "string" + usage.update({"type": output_type}) + output_value = output + + # INSERT NEW HERE + + if self.auto_remediate_function_call_output: + # attempt to remediate + output_type, output_rem = ModelCatalog().remediate_function_call_string(output) + + usage.update({"type": output_type, "remediation": True}) + output_value = output_rem + + if output_type == "string": + logger.warning(f"ONNXGenerativeModel - function call - automatic conversion of function call output " + f"failed, and attempt to remediate was not successful - {output}") + else: + logger.info(f"ONNXGenerativeModel - function call output could not be automatically converted, but " + f"remediation was successful to type -{output_type}") + + # INSERT ENDS HERE + + output_response = {"llm_response": output_value, "usage": usage} + + if get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output_value + self.usage = usage + self.final_prompt = prompt + + self.register() + + return output_response + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None): + + """ Executes stream generation inference on model. """ + + from llmware.configs import ONNXConfig + legacy = ONNXConfig().get_legacy_flag() + + # first prepare the prompt + t0 = time.time() + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + # show warning if function calling model + if self.fc_supported: + logger.warning("ONNXGenerativeModel - this is a function calling model - " + "using .inference may lead to unexpected " + "results. Recommended to use the .function_call method to " + "ensure correct prompt template packaging.") + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + # START - route to api endpoint + if self.api_endpoint: + return self.inference_over_api_endpoint(self.prompt, context=self.add_context, + inference_dict=inference_dict) + # END - route to api endpoint + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # text_prompt = prompt_final + "\n" + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + text_prompt = prompt_final + self.trailing_space + + input_tokens = self.tokenizer.encode(text_prompt) + + if legacy: + self.params.input_ids = input_tokens + + token_count = 0 + output = "" + + # adding as a state var so it can be shut down by chat app if user terminates + try: + self.generator = og.Generator(self.model, self.params) + except: + raise LLMWareException(message=f"ONNXGenerativeModel - attempt to instantiate ONNX generator with " + f"model and prompt failed. This is most likely due to an error in the " + f"installation of the onnxruntime, or a problem with loading either the " + f"model or the input tokens.") + + if not legacy: + self.generator.append_tokens(input_tokens) + + while not self.generator.is_done(): + + token_count += 1 + + if legacy: + self.generator.compute_logits() + + self.generator.generate_next_token() + + self.get_logits = False + # to get logit value + if self.get_logits: + logit = self.generator.get_output("logits") + self.register_top_logits(logit) + + new_token = self.generator.get_next_tokens()[0] + + if self.get_logits: + self.output_tokens.append(new_token) + + output += self.tokenizer_stream.decode(new_token) + + if token_count > self.max_output: + break + + yield self.tokenizer_stream.decode(new_token) + + print() + # del self.generator + self.generator = None + + llm_response = {"llm_response": output, "usage": {}} + + usage = {"input": len(input_tokens), + "output": token_count, + "total": len(input_tokens) + token_count, + "metric": "tokens", + "processing_time": time.time() - t0} + + output_response = {"llm_response": output, "usage": usage} + + if self.get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def cleanup_stream_gen_on_early_stop(self): + + """ Utility method to call if streaming interrupted early to clean up the generator. """ + + self.generator = None + return True + + def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False): + + """ Called by .inference method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + import ast + import requests + + self.prompt = prompt + self.context = context + + self.preview() + + url = self.api_endpoint + "{}".format("/") + output_raw = requests.post(url, data={"model_name": self.model_name, + "question": self.prompt, + "context": self.context, + "api_key": self.api_key, + "max_output": self.max_output, + "temperature": self.temperature}) + + try: + + output = json.loads(output_raw.text) + + # will attempt to unpack logits - but catch any exceptions and skip + if "logits" in output: + try: + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + except: + output["logits"] = [] + + # will attempt to unpack output tokens - but catch any exceptions and skip + if "output_tokens" in output: + try: + # alt: ot_int = [int(x) for x in output["output_tokens"]] + # alt: output["output_tokens"] = ot_int + output_tokens = ast.literal_eval(output["output_tokens"]) + output["output_tokens"] = output_tokens + except: + output["output_tokens"] = [] + + except: + logger.warning("warning: api inference was not successful") + output = {"llm_response": "api-inference-error", "usage": {}} + + # output inference parameters + self.llm_response = output["llm_response"] + self.usage = output["usage"] + self.final_prompt = prompt + + if "logits" in output: + self.logits = output["logits"] + if "output_tokens" in output: + self.output_tokens = output["output_tokens"] + + self.register() + + return output + + def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="", + function=None, endpoint_base=None, api_key=None, get_logits=False): + + """ Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + # send to api agent server + + import ast + import requests + + self.context = context + self.tool_type = tool_type + if model_name: + self.model_name = model_name + + self.preview() + + if endpoint_base: + self.api_endpoint = endpoint_base + + if api_key: + # e.g., "demo-test" + self.api_key = api_key + + if not params: + model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type] + mc = ModelCatalog().lookup_model_card(model_name) + if "primary_keys" in mc: + params = mc["primary_keys"] + self.primary_keys = params + + if function: + self.function = function + + self.context = context + self.prompt = prompt + + url = self.api_endpoint + "{}".format("/agent") + output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key, + "tool_type": self.tool_type, + "function": self.function, + "params": self.primary_keys, "max_output": 50, + "temperature": 0.0, "sample": False, "prompt": self.prompt, + "context": self.context, "get_logits": True}) + + try: + output = json.loads(output_raw.text) + if "logits" in output: + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + + if "output_tokens" in output: + ot_int = [int(x) for x in output["output_tokens"]] + output["output_tokens"] = ot_int + + # need to clean up logits + + except: + logger.warning(f"ONNXGenerativeModel - function call - api inference was not successful") + output = {} + + logger.info(f"ONNXGenerativeModel - executed Agent call over API endpoint - " + f"{self.model_name} - {self.function} - {output}") + + # output inference parameters + self.llm_response = output["llm_response"] + self.usage = output["usage"] + self.final_prompt = prompt + + if "logits" in output: + self.logits = output["logits"] + if "output_tokens" in output: + self.output_tokens = output["output_tokens"] + + self.register() + + return output + + +class OVGenerativeModel(BaseModel): + + """ OVGenerativeModel class implements the OpenVino generative model interface for fast inference + performance on x86 Intel architectures, including both Intel CPU and GPU. """ + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None, + prompt_wrapper=None, instruction_following=False, context_window=2048, + sample=False,max_output=100, temperature=0.0, + get_logits=False, api_endpoint=None, device="GPU", + pipeline="text2text", **kwargs): + + super().__init__() + + self.model_class = "OVGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + self.model_name = model_name + self.hf_tokenizer_name = model_name + self.model = model + self.tokenizer = tokenizer + self.sample=sample + self.get_logits=get_logits + + self.pipeline = pipeline + + if get_logits: + logger.warning(f"OVGenerativeModel - current implementation does not support " + f"get_logits option.") + self.get_logits = False + + self.auto_remediate_function_call_output = True + + # Function Call parameters + self.model_card = model_card + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.primary_keys = None + self.function = None + self.fc_supported = False + + self.cache_dir = None + + self.device = device + + if "device" in kwargs: + self.device = kwargs["device"] + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + # will look for special cache_dir set in the model card + # can be over-ridden if passed as kwarg in loading model + + if "cache_dir" in model_card: + self.cache_dir = model_card["cache_dir"] + + if "pipeline" in model_card: + self.pipeline = model_card["pipeline"] + + # will auto-detect NPU model and set device accordingly + if "npu_optimized" in model_card: + self.device = "NPU" + + # insert dynamic openvino load here + if not api_endpoint: + + global openvino + global ovg + global GLOBAL_OVG_IMPORT + global GLOBAL_OPENVINO_IMPORT + if not GLOBAL_OPENVINO_IMPORT or not GLOBAL_OVG_IMPORT: + + if not util.find_spec("openvino") or not util.find_spec("openvino_genai"): + raise LLMWareException(message="OVGenerativeModel: to use OVGenerativeModel requires " + "install of 'openvino' and 'openvino_genai' libraries. " + "Please try: `pip3 install openvino` and " + "`pip3 install openvino_genai` and confirm that your " + "hardware platform is supported.") + + if util.find_spec("openvino"): + try: + openvino = importlib.import_module("openvino") + GLOBAL_OPENVINO_IMPORT = True + except: + raise LLMWareException(message="OVGenerativeModel: could not load openvino module.") + + if openvino: + if util.find_spec("openvino_genai"): + try: + ovg = importlib.import_module("openvino_genai") + GLOBAL_OVG_IMPORT = True + except: + raise LLMWareException(message="OVGenerativeModel: could not load openvino_genai module.") + + if not openvino or not ovg: + raise LLMWareException(message="OVGenerativeModel: could not load required openvino dependencies.") + + # end dynamic import here + + # set specific parameters associated with custom models + # note - these two parameters will control how prompts are handled - model-specific + self.prompt_wrapper = prompt_wrapper + self.instruction_following = instruction_following + + if not model_card: + # safety - empty iterable rather than 'None' + model_card = {} + + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + else: + self.instruction_following = False + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + # sets trailing space default when constructing the prompt + # in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves + # performance + + self.trailing_space = "" + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + + self.model_type = None + self.config = None + + # parameters on context len + output generation + self.max_total_len = context_window + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + + # key output parameters + self.max_output=max_output + self.target_requested_output_tokens = self.max_output + + self.model_architecture = None + self.separator = "\n" + + # eos_token_id set as list to allow for more than one id + self.eos_token_id = [] + + # use_gpu parameter not used - deprecated + self.use_gpu = False + + if "cache_dir" in kwargs: + self.cache_dir = kwargs["cache_dir"] + + # no api key expected or required + self.api_key = api_key + + self.error_message = "\nUnable to identify and load model." + + # temperature settings + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at default of 0.3 + self.temperature = 0.3 + + self.add_prompt_engineering = False + self.add_context = "" + self.context = "" + self.prompt = "" + self.tool_type = "" + + self.api_endpoint = api_endpoint + self.pipe = None + + self.input_token_count = 0 + self.output_token_count = 0 + self.params = None + self.model_repo_path = None + + self.tokenizer_fn = "" + + from llmware.configs import OVConfig + + # OVConfig object provided in llmware.configs - in most cases, will not be touched, but + # exposes more options for configuration of the underlying OpenVino implementation + + # if config set to CPU - then ensure CPU execution + # note: if set, this will over-ride any other settings + + if OVConfig().get_config("device") == "CPU": + self.device = "CPU" + self.optimize_for_gpu_if_available = False + else: + self.optimize_for_gpu_if_available = OVConfig().optimize_for_gpu() + + self.generation_version = OVConfig().generation_version() + self.cache = OVConfig().get_config("cache") + self.cache_with_model = OVConfig().get_config("cache_with_model") + self.cache_custom = OVConfig().get_config("cache_custom_path") + self.apply_performance_hints = OVConfig().get_config("apply_performance_hints") + self.use_ov_tokenizer = OVConfig().get_config("use_ov_tokenizer") + self.verbose_mode = OVConfig().get_config("verbose_mode") + + self.get_token_counts = OVConfig().get_config("get_token_counts") + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + if not os.path.exists(LLMWareConfig.get_model_repo_path()): + os.mkdir(LLMWareConfig.get_model_repo_path()) + + # please note that the external tokenizer is used solely for producing + # input and output token counts - and can be switched off in OVConfig + if self.get_token_counts: + self.load_ov_external_tokenizer() + + self.performance_hints = OVConfig().get_gpu_hints() + + self.post_init() + + def load_model_for_inference (self, loading_directions, + model_card=None, pipeline=None,**kwargs): + + """ Loads OV Model from local path using loading directions. """ + + global ovg + + self.model_repo_path = loading_directions + if model_card: + self.model_card = model_card + if "pipeline" in self.model_card: + self.pipeline = self.model_card["pipeline"] + + if pipeline: + self.pipeline = pipeline + + self.validate() + + if self.device == "GPU" or (self.device == "CPU" and self.optimize_for_gpu_if_available): + + device = self.device_resolver() + if device != self.device: + # resets self.device to the resolved device + # if changed, then warning provided by resolver method + self.device = device + + if self.device == "GPU" and self.apply_performance_hints: + + for k,v in self.performance_hints.items(): + + try: + # sets GPU performance hints thru openvino core + #TODO: will evaluate if better way to construct/destruct the core object + + core = openvino.Core() + core.set_property("GPU", {k:v}) + + if self.verbose_mode: + logger.info(f"OVGenerativeModel - setting performance hint - {k} - {v}") + except: + logger.warning(f"OVGenerativeModel - unsuccessful setting performance hint - {k} - {v}") + + # default is to cache to optimize performance on subsequent loads + + # build pipeline based on type + if self.pipeline == "text2image": + self.ov_text_to_image_pipeline() + else: + # default: text2text + self.ov_text_to_text_pipeline() + + if self.verbose_mode: + logger.info(f"OVGenerativeModel - completed new pipe creation - " + f"{self.pipeline}") + + return self + + def device_resolver(self): + + """ By default, will look for 'GPU' and if device found, then will select - if no GPU, + then falls back to 'CPU'. """ + + global ovg + + try: + + # check if GPU device can be found successfully - if not, auto fallback to CPU device + + core = openvino.Core() + gpu_device_name = core.get_property("GPU", "FULL_DEVICE_NAME") + logger.info(f"OVGenerativeModel - loading - confirmed GPU device name: " + f"{gpu_device_name}") + device = "GPU" + + except: + + logger.info("OVGenerativeModel - loading - could not find GPU - setting device for CPU") + device = "CPU" + + return device + + def set_api_key(self, api_key, env_var="USER_MANAGED_OV_API_KEY"): + + """ Sets the API key - generally not needed for self-hosted OV models. """ + os.environ[env_var] = api_key + logger.info("OVGenerativeModel - added and stored OV api_key in environmental " + "variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_OV_API_KEY"): + + """ Gets API key from os.environ variable. """ + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("OVGenerativeModel - _get_api_key could not successfully " + "retrieve value from: %s ", env_var) + + return self.api_key + + def load_ov_external_tokenizer(self): + + """ Called in class constructor if OVConfig flag set to 'get_output_counts', + and will create a local instance of the tokenizer used to get the counts. """ + + if "tokenizer_local" in self.model_card: + tok_local_name = self.model_card["tokenizer_local"] + self.tokenizer = LocalTokenizer(tokenizer_fn=tok_local_name) + else: + # if no tokenizer found, then falls back to default tokenizer for 'approximate' count + self.tokenizer = Utilities().get_default_tokenizer() + + def ov_text_to_text_pipeline(self): + + """ Main entry point for instantiating models """ + + loading_directions = self.model_repo_path + + global ovg + + if self.cache: + if self.cache_with_model: + # will put the cache files co-located with the model assets + path_to_cache_dir = loading_directions + else: + path_to_cache_dir = self.cache_custom + + if self.verbose_mode: + logger.info(f"OVGenerativeModel - creating pipeline - " + f"{self.device} - {self.cache} - {path_to_cache_dir}") + + try: + + self.pipe = ovg.LLMPipeline(loading_directions, self.device, + {"CACHE_DIR": path_to_cache_dir}) + + except: + raise LLMWareException(message=f"OVGenerativeModel - attempt to instantiate LLMPipeline failed - " + f"this could be for a number of reasons, including: " + f"\n1. openvino and openvino_genai installs are not supported " + f"on this os / hardware platform." + f"\n2. the model could not found at path: {loading_directions}, or " + f"\n3. the model may not a valid OpenVino format model.") + else: + + #TODO: confirm that empty plugin instructions with no caching will work on all platforms + try: + self.pipe = ovg.LLMPipeline(loading_directions, self.device, {}) + except: + raise LLMWareException(message=f"OVGenerativeModel - attempt to instantiate LLMPipeline failed - " + f"this could be for a number of reasons, including: " + f"\n1. openvino and openvino_genai installs are not supported " + f"on this os / hardware platform." + f"\n2. the model could not found at path: {loading_directions}, or " + f"\n3. the model may not a valid OpenVino format model.") + + return True + + def ov_text_to_image_pipeline(self): + + """ Model loading entry point for new OpenVINO text_to_image + pipeline for multimedia models that generate images from text prompt. """ + + global ovg + + # auto set to GPU for faster generation + + text_encoder_device = "GPU" + unet_device = "GPU" + vae_decoder_device = "GPU" + + width = 512 + height = 512 + + self.pipe = ovg.Text2ImagePipeline(self.model_repo_path) + + self.pipe.reshape(1, height, width, self.pipe.get_generation_config().guidance_scale) + properties = {"CACHE_DIR": self.model_repo_path} + + self.pipe.compile(text_encoder_device, unet_device, vae_decoder_device, config=properties) + + return True + + def text_to_image_gen(self, prompt, image_name): + + """ Specialized generation function for image generating models. """ + + from PIL import Image + + # experiment with different step numbers + # will expose as parameter in future releases + + number_of_inference_steps_per_image = 10 + + tmp_path = LLMWareConfig().get_tmp_path() + img_path = os.path.join(tmp_path, str(image_name) + ".bmp") + + image_tensor = self.pipe.generate(prompt, + num_inference_steps=number_of_inference_steps_per_image) + + image = Image.fromarray(image_tensor.data[0]) + image.save(img_path) + + return img_path + + def ov_token_counter(self, text): + + """ Called twice in inference generation loop to get the input_token_count and + output_token_count. This step can be skipped by setting the OVConfig as follows: + + `from llmware.configs import OVConfig + OVConfig().set_config("get_token_counts", False)` + + In our testing, the performance impact is negligible, but may be different in your + environment and use case. + + If this is set to False, then no token counts will be provided in the usage totals. + """ + + if self.tokenizer: + toks = len(self.tokenizer.encode(text)) + else: + toks = 0 + + return toks + + def prompt_engineer(self, query, context, inference_dict): + + """ Applies prompt and templating preparation. """ + + # if loaded model was not pretrained on instruction_following, then skip any instructions + if not self.instruction_following: + + if context: + output = context + "\n" + query + else: + output = query + + # unlikely that there would be an 'instruct wrapping' on text, but allow for possibility + if self.prompt_wrapper: + output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper, + instruction=None) + + return output + + # move ahead to add instructions and prompt engineering + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, + context=context, + inference_dict=inference_dict) + + if prompt_dict: + prompt_engineered = prompt_dict["core_prompt"] + else: + # default case + prompt_engineered = "Please read the following text: " + context + self.separator + prompt_engineered += "Based on this text, please answer the question: " + query + self.separator + prompt_engineered += "Please answer the question only with facts provided in the materials. " \ + "If the question can not be answered in the materials, then please " \ + "respond 'Not Found.'" + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper, + instruction=None) + + return prompt_engineered + + def _generate_ov_genai(self, prompt, streamer=None): + + """ Core generation script provided by generation loop exposed in the OpenVino_GenAI library. """ + + global ovg + + if self.verbose_mode: + logger.info("OVGenerativeModel - calling openvino_genai backend in _generate_ov_genai method.") + + config = ovg.GenerationConfig() + config.max_new_tokens = self.max_output + + # prevent error in generation if sampling True and temperature is set to 0.0 + if self.sample and self.temperature == 0.0: + self.temperature = 0.2 + logger.warning(f"OVGenerativeModel - since sample is set to True, adjusting " + f"temperature from 0.0 to small value - 0.2 - to avoid error " + f"in the generation loop.") + + config.temperature = self.temperature + config.do_sample = self.sample + + # core generation step - runs generation loop on pipe with prompt and config + if streamer: + output = self.pipe.generate(prompt, config, streamer=streamer) + else: + output = self.pipe.generate(prompt, config) + + return output + + @staticmethod + def ov_default_streamer(x): + + """ Stream to console - used by default in stream method - + can be over-ridden by passing a custom streaming function to + the stream generate call. """ + + print(x, end="", flush=True) + return ovg.StreamingStatus.RUNNING + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None): + + """ Executes generation inference on model. """ + + # first prepare the prompt + self.prompt = prompt + + if add_context: + self.add_context = add_context + + self.context = self.add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + # show warning if function calling model + if self.fc_supported: + logger.warning("OVGenerativeModel - this is a function calling model - using .inference may lead " + "to unexpected results. Recommended to use the .function_call method to ensure " + "correct prompt template packaging.") + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + # START - route to api endpoint + if self.api_endpoint: + return self.inference_over_api_endpoint(self.prompt, context=self.add_context, + inference_dict=inference_dict) + # END - route to api endpoint + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # text_prompt = prompt_final + "\n" + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + text_prompt = prompt_final + self.trailing_space + + # counts the input tokens + if self.get_token_counts: + self.input_token_count = self.ov_token_counter(text_prompt) + else: + self.input_token_count = 0 + + time_start = time.time() + + # main call to inner generate function + output = self._generate_ov_genai(text_prompt) + + output_str = output + + # post-processing clean-up - stop at endoftext + eot = output_str.find("<|endoftext|>") + if eot > -1: + output_str = output_str[:eot] + + # new post-processing clean-up - stop at + eots = output_str.find("") + if eots > -1: + output_str = output_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = output_str.find(":") + if bot > -1: + output_str = output_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = output_str.find("") + if boss > -1: + output_str = output_str[boss + len(""):] + + # end - post-processing + + # counts the output tokens + if self.get_token_counts: + self.output_token_count = self.ov_token_counter(output_str) + else: + self.output_token_count = 0 + + usage = {"input": self.input_token_count, + "output": self.output_token_count, + "total": self.input_token_count + self.output_token_count, + "metric": "tokens", + "processing_time": time.time() - time_start} + + output_response = {"llm_response": output_str, "usage": usage} + + self.get_logits = False + + # output inference parameters + self.llm_response = output_str + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def fc_prompt_engineer(self, context, params=None, function=None): + + """ Prompt engineering for Function Call prompts. """ + + if not params: + params = self.primary_keys + + if not function: + function = self.function[0] + + # prepare SLIM prompt + class_str = "" + for key in params: + class_str += str(key) + ", " + if class_str.endswith(", "): + class_str = class_str[:-2] + + f = str(function) + + # key templating format for SLIM function calls + full_prompt = ": " + context + "\n" + "<{}> {} ".format(f, class_str, f) + "\n:" + + full_prompt = full_prompt + self.trailing_space + + return full_prompt + + def function_call(self, context, function=None, params=None, get_logits=False, + temperature=-99, max_output=None): + + """ This is the key inference method for SLIM models - takes a context passage and a key list + which is packaged in the prompt as the keys for the dictionary output""" + + self.context = context + + if not self.fc_supported: + logger.warning("OVGenerativeModel - loaded model does not support function calls. " + "Please either use the standard .inference method with this model, or use a " + "model that has 'function_calls' key set to True in its model card.") + return [] + + # reset and start from scratch with new function call + self.output_tokens = [] + self.logits_record = [] + + if temperature != -99: + self.temperature = temperature + + if max_output: + self.target_requested_output_tokens = max_output + + if get_logits: + logger.warning("OVGenerativeModel - current implementation does not support get_logits option.") + self.get_logits = False + + if params: + self.primary_keys = params + + if function: + self.function = function + + if not self.primary_keys: + logger.warning("OVGenerativeModel - function call - no keys provided - function call may " + "yield unpredictable results") + + self.preview() + + # START - route to api endpoint + + if self.api_endpoint: + return self.function_call_over_api_endpoint(model_name=self.model_name, + context=self.context,params=self.primary_keys, + function=self.function, + api_key=self.api_key,get_logits=self.get_logits) + + # END - route to api endpoint + + prompt = self.fc_prompt_engineer(self.context, params=self.primary_keys, function=function) + + time_start = time.time() + + # counts the input tokens + if self.get_token_counts: + self.input_token_count = self.ov_token_counter(prompt) + else: + self.input_token_count = 0 + + # main call to inner generate function + output_str = self._generate_ov_genai(prompt) + + # post-processing clean-up - stop at endoftext + eot = output_str.find("<|endoftext|>") + if eot > -1: + output_str = output_str[:eot] + + # new post-processing clean-up - stop at + eots = output_str.find("") + if eots > -1: + output_str = output_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = output_str.find(":") + if bot > -1: + output_str = output_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = output_str.find("") + if boss > -1: + output_str = output_str[boss + len(""):] + + # end - post-processing + + # counts the output tokens + if self.get_token_counts: + self.output_token_count = self.ov_token_counter(output_str) + else: + self.output_token_count = 0 + + usage = {"input": self.input_token_count, + "output": self.output_token_count, + "total": self.input_token_count + self.output_token_count, + "metric": "tokens", + "processing_time": time.time() - time_start} + + try: + output_value = ast.literal_eval(output_str) + + output_type = "dict" + + # allow for multiple valid object types - will expand over time + if isinstance(output_value,dict): output_type = "dict" + if isinstance(output_value,list): output_type = "list" + + usage.update({"type": output_type}) + + except: + # could not convert automatically to python object + output_type = "string" + usage.update({"type": output_type}) + output_value = output_str + + # auto remediate set to False - turning off this capability currently + self.auto_remediate_function_call_output = False + + if self.auto_remediate_function_call_output: + + # attempt to remediate + output_type, output_rem = ModelCatalog().remediate_function_call_string(output_str) + + usage.update({"type": output_type, "remediation": True}) + output_value = output_rem + + if output_type == "string": + logger.warning("OVGenerativeModel - automatic conversion of function call output failed, " + "and attempt to remediate was not successful - %s ", output_str) + else: + logger.info("OVGenerativeModel - function call output could not be automatically " + "converted, but remediation was successful to type - %s ", output_type) + + output_response = {"llm_response": output_value, "usage": usage} + + # get_logits - not currently implemented + if get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output_value + self.usage = usage + self.final_prompt = prompt + + self.register() + + return output_response + + def unload_model(self): + + """ Resetting the pipe removes pointer to pipeline in Python, and generally triggers a (safe) release of + the memory. WIP - will continue to evaluate effectiveness across use patterns and platforms. """ + + self.pipe = None + + return True + + def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False): + + """ Called by .inference method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + import ast + import requests + + self.prompt = prompt + self.context = context + + url = self.api_endpoint + "{}".format("/") + output_raw = requests.post(url, data={"model_name": self.model_name, + "question": self.prompt, + "context": self.context, + "api_key": self.api_key, + "max_output": self.max_output, + "temperature": self.temperature}) + + try: + + output = json.loads(output_raw.text) + + # will attempt to unpack logits - but catch any exceptions and skip + if "logits" in output: + try: + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + except: + output["logits"] = [] + + # will attempt to unpack output tokens - but catch any exceptions and skip + if "output_tokens" in output: + try: + # ot_int = [int(x) for x in output["output_tokens"]] + # output["output_tokens"] = ot_int + output_tokens = ast.literal_eval(output["output_tokens"]) + output["output_tokens"] = output_tokens + except: + output["output_tokens"] = [] + + except: + logger.warning("OVGenerativeModel - api inference was not successful") + output = {"llm_response": "api-inference-error", "usage": {}} + + # output inference parameters + self.llm_response = output["llm_response"] + self.usage = output["usage"] + self.final_prompt = prompt + + if "logits" in output: + self.logits = output["logits"] + if "output_tokens" in output: + self.output_tokens = output["output_tokens"] + + self.register() + + return output + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None, streamer=None): + + """ Executes stream generation inference on model. + + NOTE: operates differently than other stream methods in LLMWare - + the method is not a generator, but rather the streaming update is + provided through passing a streamer function to the OpenVINO + backend - which will be called at each step of the generation + cycle. + + Sample call: + + # will automatically use default streamer to print to console + response = model.stream('Where is Paris?') + + # pass a custom streaming function + response = model.stream('Where is Rome?', streamer=my_streamer) + + Streamer function example: .ov_default_streamer in this model class + + """ + + # first prepare the prompt + self.prompt = prompt + + if add_context: + self.add_context = add_context + + self.context = self.add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + # show warning if function calling model + if self.fc_supported: + logger.warning("OVGenerativeModel - this is a function calling model - using .inference may lead " + "to unexpected results. Recommended to use the .function_call method to ensure " + "correct prompt template packaging.") + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + # START - route to api endpoint + if self.api_endpoint: + return self.inference_over_api_endpoint(self.prompt, context=self.add_context, + inference_dict=inference_dict) + # END - route to api endpoint + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + text_prompt = prompt_final + self.trailing_space + + # counts the input tokens + if self.get_token_counts: + self.input_token_count = self.ov_token_counter(text_prompt) + else: + self.input_token_count = 0 + + time_start = time.time() + + # main call to inner generate function + if not streamer: + streamer = self.ov_default_streamer + + output = self._generate_ov_genai(text_prompt, streamer=streamer) + + output_str = output + + # post-processing clean-up - stop at endoftext + eot = output_str.find("<|endoftext|>") + if eot > -1: + output_str = output_str[:eot] + + # new post-processing clean-up - stop at + eots = output_str.find("") + if eots > -1: + output_str = output_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = output_str.find(":") + if bot > -1: + output_str = output_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = output_str.find("") + if boss > -1: + output_str = output_str[boss + len(""):] + + # end - post-processing + + # counts the output tokens + if self.get_token_counts: + self.output_token_count = self.ov_token_counter(output_str) + else: + self.output_token_count = 0 + + usage = {"input": self.input_token_count, + "output": self.output_token_count, + "total": self.input_token_count + self.output_token_count, + "metric": "tokens", + "processing_time": time.time() - time_start} + + output_response = {"llm_response": output_str, "usage": usage} + + self.get_logits = False + + # output inference parameters + self.llm_response = output_str + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="", + function=None, endpoint_base=None, api_key=None, get_logits=False): + + """ Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + # send to api agent server + + self.context = context + self.tool_type = tool_type + self.prompt = prompt + + import ast + import requests + + if endpoint_base: + self.api_endpoint = endpoint_base + + if api_key: + # e.g., "demo-test" + self.api_key = api_key + + if not params: + + self.model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type] + mc = ModelCatalog().lookup_model_card(self.model_name) + if "primary_keys" in mc: + params = mc["primary_keys"] + self.primary_keys = params + + if function: + self.function = function + + self.context = context + + self.preview() + + url = self.api_endpoint + "{}".format("/agent") + output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key, + "tool_type": self.tool_type, + "function": self.function, "params": self.primary_keys, "max_output": 50, + "temperature": 0.0, "sample": False, "prompt": self.prompt, + "context": self.context, "get_logits": True}) + + try: + # output = ast.literal_eval(output_raw.text) + output = json.loads(output_raw.text) + if "logits" in output: + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + + if "output_tokens" in output: + ot_int = [int(x) for x in output["output_tokens"]] + output["output_tokens"] = ot_int + + except: + logger.warning("OVGenerativeModel - api inference was not successful") + output = {} + + logger.info(f"OVGenerativeModel - executed Agent call over API endpoint - " + f"{model_name} - {function} - {output}") + + # output inference parameters + self.llm_response = output["llm_response"] + self.usage = output["usage"] + self.final_prompt = prompt + + if "logits" in output: + self.logits = output["logits"] + if "output_tokens" in output: + self.output_tokens = output["output_tokens"] + + self.register() + + return output + + +class OVVisionGenerativeModel(BaseModel): + + """ OVVisionGenerativeModel class implements the OpenVino generative model interface for fast inference + performance on x86 Intel architectures, including both Intel CPU and GPU. """ + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None, + prompt_wrapper=None, instruction_following=False, context_window=2048, + sample=False,max_output=100, temperature=0.0, + get_logits=False, api_endpoint=None, device="GPU", + pipeline="image2text", **kwargs): + + super().__init__() + + self.model_class = "OVVisionGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + self.model_name = model_name + self.hf_tokenizer_name = model_name + self.model = model + self.tokenizer = tokenizer + self.sample=sample + self.get_logits=get_logits + + self.pipeline = pipeline + + if get_logits: + logger.warning(f"OVGenerativeModel - current implementation does not support " + f"get_logits option.") + self.get_logits = False + + self.auto_remediate_function_call_output = True + + # Function Call parameters + self.model_card = model_card + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.primary_keys = None + self.function = None + self.fc_supported = False + + self.cache_dir = None + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + # will look for special cache_dir set in the model card + # can be over-ridden if passed as kwarg in loading model + + if "cache_dir" in model_card: + self.cache_dir = model_card["cache_dir"] + + if "pipeline" in model_card: + self.pipeline = model_card["pipeline"] + + # insert dynamic openvino load here + if not api_endpoint: + + global openvino + global ovg + global GLOBAL_OVG_IMPORT + global GLOBAL_OPENVINO_IMPORT + + if not GLOBAL_OPENVINO_IMPORT or not GLOBAL_OVG_IMPORT: + + if not util.find_spec("openvino") or not util.find_spec("openvino_genai"): + raise LLMWareException(message="OVGenerativeModel: to use OVGenerativeModel requires " + "install of 'openvino' and 'openvino_genai' libraries. " + "Please try: `pip3 install openvino` and " + "`pip3 install openvino_genai` and confirm that your " + "hardware platform is supported.") + + if util.find_spec("openvino"): + try: + openvino = importlib.import_module("openvino") + GLOBAL_OPENVINO_IMPORT = True + except: + raise LLMWareException(message="OVGenerativeModel: could not load openvino module.") + + if openvino: + if util.find_spec("openvino_genai"): + try: + ovg = importlib.import_module("openvino_genai") + GLOBAL_OVG_IMPORT = True + except: + raise LLMWareException(message="OVGenerativeModel: could not load openvino_genai module.") + + if not openvino or not ovg: + raise LLMWareException(message="OVGenerativeModel: could not load required openvino dependencies.") + + # end dynamic import here + + # set specific parameters associated with custom models + # note - these two parameters will control how prompts are handled - model-specific + self.prompt_wrapper = prompt_wrapper + self.instruction_following = instruction_following + + if not model_card: + # safety - empty iterable rather than 'None' + model_card = {} + + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + else: + self.instruction_following = False + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + # sets trailing space default when constructing the prompt + # in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves + # performance + + self.trailing_space = "" + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + + self.model_type = None + self.config = None + + # parameters on context len + output generation + self.max_total_len = context_window + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + + # key output parameters + self.max_output=max_output + self.target_requested_output_tokens = self.max_output + + self.model_architecture = None + self.separator = "\n" + + # eos_token_id set as list to allow for more than one id + self.eos_token_id = [] + + # use_gpu parameter not used - deprecated + self.use_gpu = False + + self.device = device + + if "device" in kwargs: + self.device = kwargs["device"] + + if "cache_dir" in kwargs: + self.cache_dir = kwargs["cache_dir"] + + # no api key expected or required + self.api_key = api_key + + self.error_message = "\nUnable to identify and load model." + + # temperature settings + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at default of 0.3 + self.temperature = 0.3 + + self.add_prompt_engineering = False + self.add_context = "" + self.context = "" + self.prompt = "" + self.tool_type = "" + + self.api_endpoint = api_endpoint + self.pipe = None + + self.input_token_count = 0 + self.output_token_count = 0 + self.params = None + self.model_repo_path = None + + self.tokenizer_fn = "" + + from llmware.configs import OVConfig + + # OVConfig object provided in llmware.configs - in most cases, will not be touched, but + # exposes more options for configuration of the underlying OpenVino implementation + + # if config set to CPU - then ensure CPU execution + if OVConfig().get_config("device") == "CPU": + self.device = "CPU" + self.optimize_for_gpu_if_available = False + else: + self.optimize_for_gpu_if_available = OVConfig().optimize_for_gpu() + + self.generation_version = OVConfig().generation_version() + self.cache = OVConfig().get_config("cache") + self.cache_with_model = OVConfig().get_config("cache_with_model") + self.cache_custom = OVConfig().get_config("cache_custom_path") + self.apply_performance_hints = OVConfig().get_config("apply_performance_hints") + self.use_ov_tokenizer = OVConfig().get_config("use_ov_tokenizer") + self.verbose_mode = OVConfig().get_config("verbose_mode") + + self.get_token_counts = OVConfig().get_config("get_token_counts") + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + if not os.path.exists(LLMWareConfig.get_model_repo_path()): + os.mkdir(LLMWareConfig.get_model_repo_path()) + + # please note that the external tokenizer is used solely for producing + # input and output token counts - and can be switched off in OVConfig + if self.get_token_counts: + self.load_ov_external_tokenizer() + + self.performance_hints = OVConfig().get_gpu_hints() + + self.post_init() + + def load_model_for_inference (self, loading_directions, + model_card=None, pipeline=None,**kwargs): + + """ Loads OV Model from local path using loading directions. """ + + global ovg + + self.model_repo_path = loading_directions + if model_card: + self.model_card = model_card + + self.validate() + + if self.device == "GPU" or (self.device == "CPU" and self.optimize_for_gpu_if_available): + + device = self.device_resolver() + if device != self.device: + # resets self.device to the resolved device + # if changed, then warning provided by resolver method + self.device = device + + if self.device == "GPU" and self.apply_performance_hints: + + for k,v in self.performance_hints.items(): + + try: + # sets GPU performance hints thru openvino core + core = openvino.Core() + core.set_property("GPU", {k:v}) + + if self.verbose_mode: + logger.info(f"OVVisionGenerativeModel - setting performance hint - {k} - {v}") + except: + logger.warning(f"OVVisionGenerativeModel - unsuccessful setting performance hint - {k} - {v}") + + # default is to cache to optimize performance on subsequent loads + + properties = {"CACHE_DIR": self.model_repo_path} + + self.pipe = ovg.VLMPipeline(self.model_repo_path, self.device,**properties) + + if self.verbose_mode: + logger.info(f"OVVisionGenerativeModel - completed new pipe creation - " + f"{self.model_name} - on device {self.device}") + + return self + + def device_resolver(self): + + """ By default, will look for 'GPU' and if device found, then will select - if no GPU, + then falls back to 'CPU'. """ + + global ovg + + try: + + # check if GPU device can be found successfully - if not, auto fallback to CPU device + + core = openvino.Core() + gpu_device_name = core.get_property("GPU", "FULL_DEVICE_NAME") + logger.info(f"OVVisionGenerativeModel - loading - confirmed GPU device name: " + f"{gpu_device_name}") + device = "GPU" + + except: + + logger.info("OVVisionGenerativeModel - loading - could not find GPU - setting device for CPU") + device = "CPU" + + return device + + def load_ov_external_tokenizer(self): + + """ Called in class constructor if OVConfig flag set to 'get_output_counts', + and will create a local instance of the tokenizer used to get the counts. """ + + if "tokenizer_local" in self.model_card: + tok_local_name = self.model_card["tokenizer_local"] + self.tokenizer = LocalTokenizer(tokenizer_fn=tok_local_name) + else: + # if no tokenizer found, then falls back to default tokenizer for 'approximate' count + self.tokenizer = Utilities().get_default_tokenizer() + + def inference(self, prompt, image_path, inference_dict=None): + """ Implemented as stream without a streamer function. """ + + return self.stream(prompt,image_path, inference_dict=inference_dict, + streamer=None, no_stream=True) + + def stream(self, prompt, image_path, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None, streamer=None,no_stream=False): + + """ Executes stream generation inference on model. + + NOTE: operates differently than other stream methods in LLMWare - + the method is not a generator, but rather the streaming update is + provided through passing a streamer function to the OpenVINO + backend - which will be called at each step of the generation + cycle. + + Sample call: + + # will automatically use default streamer to print to console + response = model.stream('Describe this image', 'C:\\Users\\...') + + # pass a custom streaming function + response = model.stream('Describe this image' 'C:\\Users\\...', streamer=my_streamer) + + Streamer function example: .ov_default_streamer in this model class + + """ + + # first prepare the prompt + self.prompt = prompt + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + text_prompt = self.prompt + + # counts the input tokens + if self.get_token_counts: + self.input_token_count = self.ov_token_counter(text_prompt) + else: + self.input_token_count = 0 + + time_start = time.time() + + # prepares the image as tensor + from PIL import Image + pic = Image.open(image_path).convert("RGB") + image_data = np.array(pic)[None] + images = [openvino.Tensor(image_data)] + + # main call to inner generate function + if not streamer and not no_stream: + streamer = self.ov_default_streamer + + output = self._generate_ov_genai(text_prompt, + image=images, + streamer=streamer) + + output_str = output + + self.output_token_count = 0 + + usage = {"input": self.input_token_count, + "output": self.output_token_count, + "total": self.input_token_count + self.output_token_count, + "metric": "tokens", + "processing_time": time.time() - time_start} + + output_response = {"llm_response": output_str, "usage": usage} + + self.get_logits = False + + # output inference parameters + self.llm_response = output_str + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def ov_token_counter(self, text): + + """ Called twice in inference generation loop to get the input_token_count and + output_token_count. This step can be skipped by setting the OVConfig as follows: + + `from llmware.configs import OVConfig + OVConfig().set_config("get_token_counts", False)` + + In our testing, the performance impact is negligible, but may be different in your + environment and use case. + + If this is set to False, then no token counts will be provided in the usage totals. + """ + + if self.tokenizer: + toks = len(self.tokenizer.encode(text)) + else: + toks = 0 + + return toks + + def prompt_engineer(self, query, context, inference_dict): + """ Implemented by openvino_genai module """ + pass + + def _generate_ov_genai(self, prompt, image=None, streamer=None): + + """ Core generation script provided by generation loop exposed in the OpenVino_GenAI library. """ + + global ovg + + config = ovg.GenerationConfig() + config.max_new_tokens = self.max_output + + self.sample=False + self.temperature =0.0 + + # prevent error in generation if sampling True and temperature is set to 0.0 + if self.sample and self.temperature == 0.0: + self.temperature = 0.2 + logger.warning(f"OVVisionGenerativeModel - since sample is set to True, adjusting " + f"temperature from 0.0 to small value - 0.2 - to avoid error " + f"in the generation loop.") + + config.temperature = self.temperature + config.do_sample = self.sample + + logger.info("OVVisionGenerativeModel - _generate_ov_genai - " + f"do_sample is {self.sample} with temperature - {self.temperature}") + + # core generation step - runs generation loop on pipe with prompt and config + + if image: + output = self.pipe.generate(prompt,image,config, streamer=streamer) + else: + if streamer: + output = self.pipe.generate(prompt, config, streamer=streamer) + else: + output = self.pipe.generate(prompt, config) + + # need to unpack the output + text_output = "" + + if output: + if hasattr(output, "texts"): + text_output = output.texts + + return text_output + + @staticmethod + def ov_default_streamer(x): + + """ Stream to console - used by default in stream method - + can be over-ridden by passing a custom streaming function to + the stream generate call. """ + + print(x, end="", flush=True) + return ovg.StreamingStatus.RUNNING + + def unload_model(self): + + """ Resetting the pipe removes pointer to pipeline in Python, and generally triggers a (safe) release of + the memory. WIP - will continue to evaluate effectiveness across use patterns and platforms. """ + + self.pipe = None + + return True + + +class OpenChatModel(BaseModel): + + """ OpenChatModel class implements the OpenAI prompt API and is intended for use with OpenChat compatible + inference servers """ + + def __init__(self, model_name=None, model_card=None, context_window=4000,prompt_wrapper=None, api_key="not_used", + **kwargs): + + super().__init__(**kwargs) + + # expected to take config parameters from model card + self.api_key = api_key + self.model_name = model_name + self.model_card = model_card + + self.model_class = "OpenChatModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + # by default, will use the 'chat' open interface, but alternative is 'completion' api + self.model_type = "chat" + + # assume that prompt_wrapper is set in the model card configuration + self.prompt_wrapper = prompt_wrapper + + # this is the key parameter that needs to be configured to pass to open chat inference server + self.api_base = "" + + if self.model_card: + + if "model_type" in self.model_card: + self.model_type = self.model_card["model_type"] + + if "api_base" in self.model_card: + self.api_base = self.model_card["api_base"] + + if "prompt_wrapper" in self.model_card: + self.prompt_wrapper = self.model_card["prompt_wrapper"] + + self.error_message = "\nUnable to connect to OpenChat Model. Please try again later." + + self.separator = "\n" + + # assume input (50%) + output (50%) + self.max_total_len = context_window + self.max_input_len = int(context_window * 0.5) + self.llm_max_output_len = int(context_window * 0.5) + + # inference settings + self.temperature = 0.7 + self.target_requested_output_tokens = 100 + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + + # new post_init check + self.post_init() + + def set_api_key (self, api_key, env_var="USER_MANAGED_OPEN_CHAT_API_KEY"): + + """ Utility method to set API key if needed. """ + + # set api_key + os.environ[env_var] = api_key + logger.info("update: added and stored OpenChat api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key (self, env_var="USER_MANAGED_OPEN_CHAT_API_KEY"): + + """ Utility method to get API key if needed. """ + + # not expected to use api_key - so may be empty - handled in inference separately + self.api_key = os.environ.get(env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Gets GPT2 tokenizer for fast approximate token counting. """ + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + + return len(toks) + + def prompt_engineer_chat(self, query, context, inference_dict=None): + + """ Creates Prompt Template for Chat Interaction. """ + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, context=context, + inference_dict=inference_dict) + + system_message = prompt_dict["prompt_card"]["system_message"] + if not system_message: + system_message = "You are a helpful assistant." + + core_prompt = prompt_dict["core_prompt"] + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + core_prompt = PromptCatalog().apply_prompt_wrapper(core_prompt, self.prompt_wrapper, instruction=None) + + messages = [ + {"role": "system", "content": system_message}, + {"role": "user", "content": core_prompt} + ] + + return messages + + def prompt_engineer_completion (self, query, context, inference_dict=None): + + """ Creates Prompt for 'Completion' style interface. """ + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, context=context, + inference_dict=inference_dict) + + core_prompt = prompt_dict["core_prompt"] + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + core_prompt = PromptCatalog().apply_prompt_wrapper(core_prompt, self.prompt_wrapper, instruction=None) + + return core_prompt + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes inference on the Model. Required input is a text prompt. Optional parameters include + an 'add_context' to be used as a source in the prompt, and assembled according to the prompt + engineering style (e.g., add_prompt_engineering). An optional inference_dict can include other optional + parameters such as temperature and max_tokens. If an API key is required, it can be passed here, or + will be picked up through the appropriate os.environ variable """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # api_key + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + # call to preview (not implemented by default) + self.preview() + + # expect that .api_base will route to local open chat inference server + # -- assumed that *** api_key likely not used *** + # -- in openai >= 1.0: .api_base replaced with 'base_url' attribute + + try: + from openai import OpenAI + except ImportError: + raise DependencyNotInstalledException("openai >= 1.0") + + if not self.api_key: + client = OpenAI(api_key="not-used",base_url=self.api_base) + else: + client = OpenAI(api_key=self.api_key,base_url=self.api_base) + + # default case - pass the prompt received without change + prompt_enriched = self.prompt + + usage = {} + time_start = time.time() + + try: + + if self.model_type == "chat": + + messages = self.prompt_engineer_chat(prompt_enriched, self.add_context, inference_dict) + + # using openai >1.0 api -> create client object, and output is pydantic, not dicts + + response = client.chat.completions.create(model=self.model_name,messages=messages, + max_tokens=self.target_requested_output_tokens) + + """ assume 'minimal' api output conformance with OpenAI """ + + text_out = response.choices[0].message.content + + """ note: some openchat api do not support providing usage output consistent with OpenAI API """ + + pt = 0 + ct = 0 + tt = 0 + + """ best effort to gather usage data if conforms with OpenAI """ + + if hasattr(response, "usage"): + + if hasattr(response.usage, "prompt_tokens"): + pt = response.usage.prompt_tokens + + if hasattr(response.usage, "completion_tokens"): + ct = response.usage.completion_tokens + + if hasattr(response.usage, "total_tokens"): + tt = response.usage.total_tokens + + usage = {"input": pt, + "output": ct, + "total": tt, + "metric": "tokens", + "processing_time": time.time() - time_start} + + else: + + # traditional completion 'instruct gpt' models + + prompt_enriched = self.prompt_engineer_completion(prompt_enriched, + self.add_context, + inference_dict=inference_dict) + + prompt_final = prompt_enriched + + text_prompt = prompt_final + self.separator + + response = client.completions.create(model=self.model_name, prompt=text_prompt, + temperature=self.temperature, + max_tokens=self.target_requested_output_tokens) + + """ assume 'minimal' api output conformance with OpenAI """ + + text_out = response.choices[0].text + + """ note: some openchat api do not support providing usage output consistent with OpenAI API """ + + pt = 0 + ct = 0 + tt = 0 + + """ best effort to gather usage data if conforms with OpenAI API """ + + if hasattr(response, "usage"): + + if hasattr(response.usage, "prompt_tokens"): + pt = response.usage.prompt_tokens + + if hasattr(response.usage, "completion_tokens"): + ct = response.usage.completion_tokens + + if hasattr(response.usage, "total_tokens"): + tt = response.usage.total_tokens + + usage = {"input": pt, + "output": ct, + "total": tt, + "metric": "tokens", + "processing_time": time.time() - time_start} + + except Exception as e: + + text_out = "/***ERROR***/" + usage = {"input":0, "output":0, "total":0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error(f"Open Chat model inference produced error - {e}") + + output_response = {"llm_response": text_out, "usage": usage} + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class OllamaModel(BaseModel): + + """ OllamaModel class implements the Ollama model prompt API and is intended for use in building + RAG pipelines while using a Ollama endpoint primarily for rapid local prototyping. """ + + def __init__(self, model_name=None, model_card=None, context_window=4000,prompt_wrapper=None, api_key="not_used", + **kwargs): + + super().__init__(**kwargs) + + self.model_class = "OllamaModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + # default ollama specific settings + # self.uri = "http://localhost:11434/api/" + self.host = "localhost" + self.port = 11434 + self.model_name = "llama2" + self.model_type = "chat" + self.stream_mode = False + self.raw_mode = False + + # expected to take config parameters from model card + self.api_key = api_key + self.model_name = model_name + self.model_card = model_card + + # assume that prompt_wrapper is set in the model card configuration + self.prompt_wrapper = prompt_wrapper + + if self.model_card: + + if "model_name" in self.model_card: + self.model_name = self.model_card["model_name"] + + if "model_type" in self.model_card: + self.model_type = self.model_card["model_type"] + + if "host" in self.model_card: + self.host = self.model_card["host"] + + if "port" in self.model_card: + self.port = self.model_card["port"] + + if "prompt_wrapper" in self.model_card: + self.prompt_wrapper = self.model_card["prompt_wrapper"] + + if "raw_mode" in self.model_card: + self.raw_mode = self.model_card["raw_mode"] + + if "stream_mode" in self.model_card: + self.stream_mode = self.model_card["stream_mode"] + + self.error_message = f"\nUnable to connect to Ollama Model. Please check that Ollama is running"\ + f"at {self.host}:{self.port}" + + self.separator = "\n" + + # assume input (50%) + output (50%) + self.max_total_len = context_window + self.max_input_len = int(context_window * 0.5) + self.llm_max_output_len = int(context_window * 0.5) + + # inference settings -> not used as generation handled by Ollama inference + self.temperature = 0.7 + self.target_requested_output_tokens = 100 + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + + # self.uri = "http://localhost:11434/api/" + self.uri = f"http://{self.host}:{self.port}/api/" + + self.post_init() + + def set_api_key (self, api_key, env_var="USER_MANAGED_OLLAMA_API_KEY"): + + """ Utility method to store api_key in os.environ variable. """ + + # set api_key + os.environ[env_var] = api_key + logger.info("update: added and stored Ollama api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key (self, env_var="USER_MANAGED_OLLAMA_API_KEY"): + + """ Utility method to get api_key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Uses default GPT2 tokenizer for fast, approximate token count, if needed. """ + + # note: this is an approximation for counting the input tokens using a default tokenizer + # --to get 100% accurate, need to use the tokenizer being applied on the 'ollama' decoding + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + + return len(toks) + + def prompt_engineer (self, query, context, inference_dict=None): + + """ Builds prompt by assembling query, context and applying the selected prompt style. """ + + # by default, this will construct a very basic prompt, concatenating the + # query + context with a basic instruction + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, context=context, + inference_dict=inference_dict) + + core_prompt = prompt_dict["core_prompt"] + + # Ollama will handle the prompt wrap templating, unless self.raw_mode = True + if self.raw_mode: + if self.prompt_wrapper: + core_prompt = PromptCatalog().apply_prompt_wrapper(core_prompt, self.prompt_wrapper, + instruction=None) + + return core_prompt + + def discover_models(self): + + """ Calls Ollama endpoint for discovery of available models and their locations. """ + + response = requests.get(self.uri+"tags") + + logger.info("update: OllamaModel - discover_models - %s ", response.text) + + output = json.loads(response.text) + + return output + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ In typical case with raw_mode = False, then no prompt engineering, just apply a basic + assembly of the prompt and context. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # call to preview hook (not implemented by default) + self.preview() + + # default case - pass the prompt received without change + prompt_enriched = self.prompt + + usage = {} + + time_start = time.time() + + try: + + # assumes 'chat' api by default + + if self.model_type == "chat": + + full_prompt = self.prompt_engineer(prompt_enriched, self.add_context, inference_dict) + + messages = [{"role": "user", "content": full_prompt}] + uri = self.uri + "chat" + + response = requests.post(uri, + json={"model": self.model_name, + "messages": messages, "stream": self.stream_mode}) + + logger.info("update: OllamaModel response - chat - %s ", response.text) + + output = json.loads(response.text) + + text_out = output["message"]["content"] + + pt = 0 + ct = 0 + tt = 0 + + """ best effort to gather usage data """ + + if "eval_count" in output: + ct = output["eval_count"] + tt += ct + + pt = self.token_counter(full_prompt) + + tt += pt + + usage = {"input": pt, + "output": ct, + "total": tt, + "metric": "tokens", + "processing_time": time.time() - time_start} + + else: + + # traditional completion 'instruct gpt' api + + prompt_enriched = self.prompt_engineer(prompt_enriched, self.add_context, + inference_dict=inference_dict) + + prompt_final = prompt_enriched + self.separator + + params = {"model": self.model_name, "prompt": prompt_final, "stream": self.stream_mode} + + # response = requests.post("http://localhost:11434/api/generate", json=params) + response = requests.post(self.uri+"generate", json=params) + + output = json.loads(response.text) + + text_out = output["response"] + + pt = 0 + ct = 0 + tt = 0 + + """ best effort to gather usage data if conforms with OpenAI API """ + + if "eval_count" in output: + + ct = output["eval_count"] + tt += ct + + pt = self.token_counter(prompt_final) + tt += pt + + usage = {"input": pt, + "output": ct, + "total": tt, + "metric": "tokens", + "processing_time": time.time() - time_start} + + except Exception as e: + + text_out = "/***ERROR***/" + usage = {"input":0, "output":0, "total":0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error(f"error: Ollama model inference produced error - {e}") + + output_response = {"llm_response": text_out, "usage": usage} + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class OpenAIGenModel(BaseModel): + + """ OpenAIGenModel class implements the OpenAI API for its generative decoder models. """ + + def __init__(self, model_name=None, api_key=None, context_window=32768, + max_output=1000,temperature=0.0, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "OpenAIGenModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + self.api_key = api_key + self.model_name = model_name + + self.error_message = "\nUnable to connect to OpenAI. Please try again later." + + self.separator = "\n" + + # assume input (50%) + output (50%) + self.max_total_len = context_window + self.max_input_len = int(context_window * 0.5) + self.llm_max_output_len = int(context_window * 0.5) + + # inference settings + if temperature >= 0.0: + self.temperature = temperature + else: + self.temperature = 0.0 + + self.target_requested_output_tokens = max_output + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + self.context = "" + + # provides option to pass custom openai_client to model class at inference time + self.openai_client = None + + if "model_card" in kwargs: + self.model_card = kwargs["model_card"] + else: + self.model_card = {} + + self.post_init() + + def set_api_key (self, api_key, env_var="OPENAI_API_KEY"): + + """ Utility method to set the API key in os.environ variable. """ + + # set api_key + os.environ[env_var] = api_key + logger.info(f"OpenAIGenModel - added and stored OpenAI api_key in environmental variable- {env_var}") + + return self + + def _get_api_key (self, env_var="OPENAI_API_KEY"): + + """ Utility method to get the API key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error(f"OpenAIGenModel - _get_api_key could not successfully retrieve " + f"value from: {env_var}") + + return self.api_key + + def token_counter(self, text_sample): + + """ Fast, approximate token counting using GPT2 tokenizer. """ + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + + return len(toks) + + def prompt_engineer_chatgpt3(self, query, context, inference_dict=None): + + """ Builds prompt in ChatGPT format. """ + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, context=context, + inference_dict=inference_dict) + + system_message = prompt_dict["prompt_card"]["system_message"] + if not system_message: + system_message = "You are a helpful assistant." + + core_prompt = prompt_dict["core_prompt"] + + messages = [ + {"role": "system", "content": system_message}, + {"role": "user", "content": core_prompt} + ] + + return messages + + def prompt_engineer (self, query, context, inference_dict=None): + + """ Builds Prompt in traditional 'completion' style. """ + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, context=context, + inference_dict=inference_dict) + + core_prompt = prompt_dict["core_prompt"] + + return core_prompt + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes inference on OpenAI Model. Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if "openai_client" in inference_dict: + self.openai_client = inference_dict["openai_client"] + + from llmware.configs import OpenAIConfig + + if not self.openai_client: + azure_client = OpenAIConfig().get_azure_client() + else: + azure_client = self.openai_client + + # api_key + if api_key: + self.api_key = api_key + + if not self.api_key: + if not azure_client: + self.api_key = self._get_api_key() + + if not self.api_key and not azure_client: + raise LLMWareException(message="OpenAIGenModel: no api_key found for OpenAI. This can be set as " + "an environment variable with: os.environ['OPENAI_API_KEY'] = '...'") + + # call to preview hook (not implemented by default) + self.preview() + + # default case - pass the prompt received without change + prompt_enriched = self.prompt + + try: + from openai import OpenAI + except ImportError: + raise DependencyNotInstalledException("openai >= 1.0") + + usage = {} + time_start = time.time() + + try: + + if self.model_name in ["gpt-4o", "o4-mini"]: + + # PATH #1 - the new 'responses' endpoint + + messages = self.prompt_engineer_chatgpt3(prompt_enriched, self.add_context, inference_dict) + + # updated OpenAI client to >v1.0 API - create client, and returns pydantic objects + + if not azure_client: + client = OpenAI(api_key=self.api_key) + model_name = self.model_name + else: + logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig") + + client = azure_client + + # adapt model name for azure, e.g., replace(".", "") + model_name = OpenAIConfig().get_azure_model_name(self.model_name) + + response = client.responses.create(model=model_name,input=messages,) + + text_out = response.output_text + + usage = {"input": response.usage.input_tokens, + "output": response.usage.output_tokens, + "total": response.usage.total_tokens, + "metric": "tokens", + "processing_time": time.time() - time_start} + + elif self.model_name in ["gpt-5.2-pro", "gpt-5.2", "gpt-5-mini", "gpt-5-nano", "gpt-4.1"]: + + # PATH #2 - 'main' chatgpt-style chat completions endpoint + + messages = self.prompt_engineer_chatgpt3(prompt_enriched, self.add_context, inference_dict) + + # updated OpenAI client to >v1.0 API - create client, and returns pydantic objects + + if not azure_client: + client = OpenAI(api_key=self.api_key) + model_name = self.model_name + + else: + + logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig") + + client = azure_client + + # adapt model name for azure, e.g., replace(".", "") + model_name = OpenAIConfig().get_azure_model_name(self.model_name) + + # note: max_tokens deprecated for max_output_tokens -> but not supported for 'o' models + + response = client.chat.completions.create(model=model_name, messages=messages) + + text_out = response.choices[0].message.content + + usage = {"input": response.usage.prompt_tokens, + "output": response.usage.completion_tokens, + "total": response.usage.total_tokens, + "metric": "tokens", + "processing_time": time.time() - time_start} + + else: + + # PATH #3 - openai traditional 'instruct gpt' completion models + + prompt_enriched = self.prompt_engineer(prompt_enriched, self.add_context, inference_dict=inference_dict) + + prompt_final = prompt_enriched + + text_prompt = prompt_final + self.separator + + azure_client = OpenAIConfig().get_azure_client() + + if not azure_client: + client = OpenAI(api_key=self.api_key) + model_name = self.model_name + else: + + logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig") + + client = azure_client + # adapt model name for azure, e.g., replace(".", "") + model_name = OpenAIConfig().get_azure_model_name(self.model_name) + + response = client.completions.create(model=model_name, prompt=text_prompt, + temperature=self.temperature, + max_tokens=self.target_requested_output_tokens) + + text_out = response.choices[0].text + + usage = {"input": response.usage.prompt_tokens, + "output": response.usage.completion_tokens, + "total": response.usage.total_tokens, + "metric": "tokens", + "processing_time": time.time() - time_start} + + except Exception as e: + # catch error + text_out = "/***ERROR***/" + usage = {"input":0, "output":0, "total":0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error(f"OpenAIGenModel - inference produced error - {e}") + + output_response = {"llm_response": text_out, "usage": usage} + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes stream inference on OpenAI Model. + + Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. + """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if "openai_client" in inference_dict: + self.openai_client = inference_dict["openai_client"] + + from llmware.configs import OpenAIConfig + + if not self.openai_client: + azure_client = OpenAIConfig().get_azure_client() + else: + azure_client = self.openai_client + + # api_key + if api_key: + self.api_key = api_key + + if not self.api_key: + if not azure_client: + self.api_key = self._get_api_key() + + if not self.api_key and not azure_client: + raise LLMWareException(message="OpenAIGenModel: no api_key found for OpenAI. This can be set as " + "an environment variable with: os.environ['OPENAI_API_KEY'] = '...'") + + # call to preview hook (not implemented by default) + self.preview() + + # default case - pass the prompt received without change + prompt_enriched = self.prompt + + try: + from openai import OpenAI + except ImportError: + raise DependencyNotInstalledException("openai >= 1.0") + + usage = {} + time_start = time.time() + + try: + + if self.model_name in ["o1-pro", "o3-mini"]: + + # PATH #1 - the new 'responses' endpoint -> streaming not implemented yet + + raise LLMWareException(message=f"Responses API streaming not implemented for this model. To use " + f"{self.model_name}, please use the .inference method") + + elif self.model_name in ["gpt-5.2-pro", "gpt-5.2", "gpt-5-mini", "gpt-5-nano", "gpt-4.1"]: + + messages = self.prompt_engineer_chatgpt3(prompt_enriched, self.add_context, inference_dict) + + # updated OpenAI client to >v1.0 API - create client, and returns pydantic objects + + if not azure_client: + client = OpenAI(api_key=self.api_key) + model_name = self.model_name + + else: + + logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig.") + + client = azure_client + + # adapt model name for azure, e.g., replace(".", "") + model_name = OpenAIConfig().get_azure_model_name(self.model_name) + + text_out = "" + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + stream_response = client.chat.completions.create(model=model_name,messages=messages, + # max_tokens=self.target_requested_output_tokens, + stream=True) + + # implement streaming generator to yield chunk of tokens + for chunk in stream_response: + if len(chunk.choices) > 0: + token = chunk.choices[0].delta.content or "" + text_out += token + yield token + + usage = {"input": prompt_tokens, + "output": completion_tokens, + "total": prompt_tokens + completion_tokens, + "metric": "tokens", + "processing_time": time.time() - time_start} + + else: + # openai traditional 'instruct gpt' completion models + + prompt_enriched = self.prompt_engineer(prompt_enriched, self.add_context, inference_dict=inference_dict) + + prompt_final = prompt_enriched + + text_prompt = prompt_final + self.separator + + azure_client = OpenAIConfig().get_azure_client() + + if not azure_client: + client = OpenAI(api_key=self.api_key) + model_name = self.model_name + + else: + + logger.debug("OpenAIGenModel - applying custom OpenAI client from OpenAIConfig.") + + client = azure_client + model_name = OpenAIConfig().get_azure_model_name(self.model_name) + + text_out = "" + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + stream_response = client.completions.create(model=model_name, prompt=text_prompt, + temperature=self.temperature, + max_tokens=self.target_requested_output_tokens, + stream=True) + + # implement streaming generator to yield chunk of tokens + for chunk in stream_response: + if len(chunk.choices) > 0: + token = chunk.choices[0].delta.content or "" + text_out += token + yield token + + usage = {"input": prompt_tokens, + "output": completion_tokens, + "total": prompt_tokens + completion_tokens, + "metric": "tokens", + "processing_time": time.time() - time_start} + + except Exception as e: + # catch error + text_out = "/***ERROR***/" + usage = {"input":0, "output":0, "total":0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error(f"OpenAIGenModel - OpenAI model inference produced error - {e}") + + output_response = {"llm_response": text_out, "usage": usage} + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class ClaudeModel(BaseModel): + + """ ClaudeModel class implements the Anthropic Claude API for calling Anthropic models. """ + + def __init__(self, model_name=None, api_key=None, context_window=32768, + max_output=1000, temperature=0.0, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "ClaudeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + self.api_key = api_key + + if not api_key: + self.api_key = api_key + + self.model_name = model_name + + self.error_message = "\nUnable to connect to Anthropic/Claude. Please try again later." + + self.separator = "\n" + + # Claude/Anthropic model - 8000 max token context window + self.max_total_len = context_window + self.max_input_len = int(context_window * 0.5) + self.llm_max_output_len = int(context_window * 0.5) + + # inference settings + if temperature >= 0.0: + self.temperature = temperature + else: + self.temperature = 0.0 + + self.target_requested_output_tokens = max_output + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + self.instruction_following = False + self.prompt_wrapper = None + + if "model_card" in kwargs: + self.model_card = kwargs["model_card"] + else: + self.model_card = {} + + self.post_init() + + def set_api_key(self, api_key, env_var="ANTHROPIC_API_KEY"): + + """ Utility method to set the API key in os.environ variable. """ + + os.environ[env_var] = api_key + logger.info(f"ClaudeModel - added and stored ANTHROPIC api_key in environmental variable- {env_var}") + + return self + + def _get_api_key(self, env_var="ANTHROPIC_API_KEY"): + + """ Utility method to get api_key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error(f"ClaudeModel - _get_api_key could not successfully retrieve value from: {env_var}") + + return self.api_key + + def token_counter(self, text_sample): + + """ Gets GPT2 tokenizer for fast approximate token counting. """ + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + return len(toks) + + def prompt_engineer(self, query, context, inference_dict=None): + + self.instruction_following = False + self.prompt_wrapper = False + + # new + system_instruction = None + if inference_dict: + if "system_instruction" in inference_dict: + system_instruction = inference_dict["system_instruction"] + # end - new + + # if loaded model was not pretrained on instruction_following, then skip any instructions + if not self.instruction_following: + + if context: + output = context + "\n" + query + else: + output = query + + # unlikely that there would be an 'instruct wrapping' on text, but allow for possibility + if self.prompt_wrapper: + output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper, + instruction=system_instruction) + + return output + + # move ahead to add instructions and prompt engineering + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, + context=context, + inference_dict=inference_dict) + + if prompt_dict: + prompt_engineered = prompt_dict["core_prompt"] + else: + # default case + prompt_engineered = "Please read the following text: " + context + self.separator + prompt_engineered += "Based on this text, please answer the question: " + query + self.separator + prompt_engineered += "Please answer the question only with facts provided in the materials. " \ + "If the question can not be answered in the materials, then please " \ + "respond 'Not Found.'" + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper, + instruction=None) + + return prompt_engineered + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes inference on Anthropic Model. Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + if not self.api_key: + raise LLMWareException(message=f"ClaudeModel - no api key found - you can set with: " + f"os.environ['ANTHROPIC_API_KEY'] = '...'") + + # call to preview hook (not implemented by default) + self.preview() + + try: + import anthropic + except ImportError: + raise DependencyNotInstalledException("anthropic") + + client = anthropic.Client(api_key=self.api_key) + + prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict) + + time_start = time.time() + + try: + + # use messages API - older completion api is deprecated (and removed from ClaudeModel) + + message = client.messages.create(model=self.model_name, max_tokens=self.target_requested_output_tokens, + messages=[{"role": "user", "content": prompt_enriched}] ) + + text_out = message.content[0].text + input_count = message.usage.input_tokens + output_count = message.usage.output_tokens + + usage = {"input": input_count, "output": output_count, "total": input_count + output_count, + "metric": "tokens", "processing_time": time.time() - time_start} + + except Exception as e: + # this is special error code that will be picked and handled by calling function + text_out = "/***ERROR***/" + usage = {"input":0, "output":0, "total":0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error(f"ClaudeModel - inference produced error - {e}") + + output_response = {"llm_response": str(text_out), "usage": usage} + + logger.debug(f"ClaudeModel - output_response - {output_response}") + + # output inference parameters + self.llm_response = str(text_out) + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes streaming inference on Anthropic Model. Only required input is text-based prompt, + with optional parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens + configuration, and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + if not self.api_key: + raise LLMWareException(message=f"ClaudeModel - no api key found - you can set with: " + f"os.environ['ANTHROPIC_API_KEY'] = '...'") + + # call to preview hook (not implemented by default) + self.preview() + + try: + import anthropic + except ImportError: + raise DependencyNotInstalledException("anthropic") + + client = anthropic.Client(api_key=self.api_key) + + prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict) + + time_start = time.time() + + try: + + # use messages API + message = client.messages.create(model=self.model_name, max_tokens=self.target_requested_output_tokens, + messages=[{"role": "user", "content": prompt_enriched}]) + + text_out = message.content[0].text + input_count = message.usage.input_tokens + output_count = message.usage.output_tokens + + text_out = "" + prompt_tokens = 0 + completion_tokens = 0 + + with client.messages.stream( + max_tokens=self.target_requested_output_tokens, + messages=[{"role": "user", "content": prompt_enriched}], + model=self.model_name) as stream: + + for text in stream.text_stream: + # print(text, end="", flush=True) + text_out += text + yield text + + usage = {"input": input_count, "output": output_count, "total": input_count + output_count, + "metric": "tokens", "processing_time": time.time() - time_start} + + except Exception as e: + # this is special error code that will be picked and handled by calling function + text_out = "/***ERROR***/" + usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error(f"ClaudeModel inference produced error - {e}") + + output_response = {"llm_response": text_out, "usage": usage} + + logger.debug(f"ClaudeModel - output_response - {output_response}") + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class GoogleGeminiModel(BaseModel): + + """ GoogleGeminiModel class implements the current Google Gemini Model + API for calling Google Gemini models. """ + + def __init__(self, model_name=None, api_key=None, context_window=32768, + max_output=1000, temperature=0.0, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "GoogleGeminiModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + self.api_key = api_key + + if not api_key: + self.api_key = api_key + + self.model_name = model_name + + self.error_message = "\nUnable to connect to Google Gemini. Please try again later." + + self.separator = "\n" + + # Google Gemini model - 8000 max token context window + self.max_total_len = context_window + self.max_input_len = int(context_window * 0.5) + self.llm_max_output_len = int(context_window * 0.5) + + # inference settings + if temperature >= 0.0: + self.temperature = temperature + else: + self.temperature = 0.0 + + self.target_requested_output_tokens = max_output + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + self.instruction_following = False + self.prompt_wrapper = None + + self.post_init() + + def set_api_key(self, api_key, env_var="GEMINI_API_KEY"): + + """ Utility method to set the API key in os.environ variable. """ + + os.environ[env_var] = api_key + logger.info(f"GoogleGeminiModel - added and stored GOOGLE GEMINI api_key in " + f"environmental variable - {env_var}") + + return self + + def _get_api_key(self, env_var="GEMINI_API_KEY"): + + """ Utility method to get api_key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error(f"GoogleGeminiModel - _get_api_key could not successfully " + f"retrieve value from: {env_var}") + + return self.api_key + + def prompt_engineer(self, query, context, inference_dict=None): + + self.instruction_following = False + self.prompt_wrapper = False + + system_instruction = None + if inference_dict: + if "system_instruction" in inference_dict: + system_instruction = inference_dict["system_instruction"] + + # if loaded model was not pretrained on instruction_following, then skip any instructions + if not self.instruction_following: + + if context: + output = context + "\n" + query + else: + output = query + + # unlikely that there would be an 'instruct wrapping' on text, but allow for possibility + if self.prompt_wrapper: + output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper, + instruction=system_instruction) + + return output + + # move ahead to add instructions and prompt engineering + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, + context=context, + inference_dict=inference_dict) + + if prompt_dict: + prompt_engineered = prompt_dict["core_prompt"] + else: + # default case + prompt_engineered = "Please read the following text: " + context + self.separator + prompt_engineered += "Based on this text, please answer the question: " + query + self.separator + prompt_engineered += "Please answer the question only with facts provided in the materials. " \ + "If the question can not be answered in the materials, then please " \ + "respond 'Not Found.'" + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper, + instruction=None) + + return prompt_engineered + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes inference on Google Gemini Model. Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + if not self.api_key: + logger.warning("GoogleGeminiModel - inference - invoking " + "Google Gemini Generative model with no api_key") + return False + + # call to preview hook (not implemented by default) + self.preview() + + try: + from google import genai + from google.genai import types + + except ImportError: + raise DependencyNotInstalledException("google") + + client = genai.Client( + api_key=self.api_key, + http_options=types.HttpOptions(api_version='v1alpha') + ) + + prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict) + + time_start = time.time() + + try: + + response = client.models.generate_content( + model=self.model_name, contents=prompt_enriched) + + text_out = response.text + + input_count = response.usage_metadata.prompt_token_count + output_count = response.usage_metadata.total_token_count + + usage = {"input": input_count, "output": output_count, + "total": input_count + output_count, + "metric": "tokens", + "processing_time": time.time() - time_start} + + except Exception as e: + # this is special error code that will be picked and handled by calling function + text_out = "/***ERROR***/" + usage = {"input":0, "output":0, "total":0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.warning(f"GoogleGeminiModel - inference produced error - {e}") + + output_response = {"llm_response": str(text_out), "usage": usage} + + # output inference parameters + self.llm_response = str(text_out) + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + def _prep_gemini_img_file(self, image_fp): + + """ Utility function to prepare image for processing by Gemini """ + + try: + from google import genai + from google.genai import types + + except ImportError: + raise DependencyNotInstalledException("google") + + img = open(image_fp, "rb").read() + ext = image_fp.split(".")[-1] + if ext in ["jpg", "jpeg"]: + mime_type = "image/jpeg" + elif ext in ["png"]: + mime_type = "image/png" + else: + mime_type = "image/jpeg" + + img_content = types.Part.from_bytes(data=img, mime_type=mime_type) + + return img_content + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None, image_files=None, doc_files=None): + + """ Executes streaming inference on Gemini Model. Only required input is text-based prompt, + with optional parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens + configuration, and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + if not self.api_key: + raise LLMWareException("GoogleGeminiModel - no api_key found - you can set with: " + "os.environ['GEMINI_API_KEY'] = '...'") + + # call to preview hook (not implemented by default) + self.preview() + + try: + from google import genai + from google.genai import types + + except ImportError: + raise DependencyNotInstalledException("google") + + client = genai.Client( + api_key=self.api_key, + http_options=types.HttpOptions(api_version='v1alpha') + ) + + prompt_enriched = self.prompt_engineer(self.prompt,self.add_context, inference_dict=inference_dict) + + time_start = time.time() + + content = [] + content.append(prompt_enriched) + + if image_files: + for img_fp in image_files: + img_content = self._prep_gemini_img_file(img_fp) + content.append(img_content) + + try: + + for chunk in client.models.generate_content_stream(model=self.model_name, + contents=content): + yield chunk.text + + text_out = "" + prompt_tokens = 0 + completion_tokens = 0 + + usage = {"input": prompt_tokens, "output": completion_tokens, + "total": prompt_tokens + completion_tokens, + "metric": "tokens", "processing_time": time.time() - time_start} + + except Exception as e: + # this is special error code that will be picked and handled by calling function + text_out = "/***ERROR***/" + usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.warning(f"GoogleGeminiModel - streaming inference produced error - {e}") + + output_response = {"llm_response": text_out, "usage": usage} + + logger.debug(f"GoogleGeminiModel - output_response - {output_response}") + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class ONNXQNNGenerativeModel(BaseModel): + + """ONNXQNNGenerativeModel class implements the ONNX generative model API in conjunction + with QNN execution provider to access NPU on Windows Arm 64. + + note: this code and associated prepackaged models are pinned to the + following specific versions: + + -- pip install onnxruntime-qnn==1.22.2 + -- pip install onnxruntime-genai==0.9.0 + + ... built with qnn sdk 2.36.1 + ... running on Windows Arm 64 Qualcomm Snapdragon NPU + ... does not currently support Android - but is on the roadmap + + """ + + def __init__(self, model_name=None, api_key=None, model_card=None, + prompt_wrapper=None, instruction_following=False, context_window=2048, + use_gpu_if_available=True, trust_remote_code=True, sample=True, max_output=100, temperature=0.3, + get_logits=False, api_endpoint=None, **kwargs): + + super().__init__() + + self.model_class = "ONNXQNNGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + logger.info(f"ONNXQNNGenerativeModel - starting constructor with model - {model_name}") + + # pull in expected hf input + self.model_name = model_name + self.hf_tokenizer_name = model_name + self.model = None + self.tokenizer = None + self.generator = None + + self.sample = sample + self.get_logits = get_logits + self.auto_remediate_function_call_output = True + + # Function Call parameters + self.model_card = model_card + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.primary_keys = None + self.function = None + self.fc_supported = False + self.tool_type = None + self.npu_optimized = False + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + if "npu_optimized" in model_card: + self.npu_optimized = True + + # instantiate if model_name passed without actual model and tokenizer + if model_name and not api_endpoint: + + hf_repo_name = self.model_name + + if not self.model_card: + self.model_card = ModelCatalog().lookup_model_card(self.model_name) + + if self.model_card: + if "hf_repo" in self.model_card: + hf_repo_name = self.model_card["hf_repo"] + self.hf_tokenizer_name = hf_repo_name + + self.model = None + self.tokenizer = None + self.tokenizer_stream = None + + # set to defaults for HF models in Model Catalog + # this can be over-ridden post initiation if needed for custom models + self.prompt_wrapper = "human_bot" + self.instruction_following = False + + self.params = None + + # set specific parameters associated with custom models + # note - these two parameters will control how prompts are handled - model-specific + self.prompt_wrapper = "human_bot" + self.instruction_following = instruction_following + + if not model_card: + # safety - empty iterable rather than 'None' + model_card = [] + + # deprecated attribute - will be removed in future releases + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + else: + self.instruction_following = False + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + # loads onnxruntime_genai, which in turn will look for backend qnn implementation + # please ensure that onnxruntime_qnn has been imported into the project + # onnxruntime_qnn==1.22.2 + + global GLOBAL_ONNX_GENAI_RUNTIME + + if not GLOBAL_ONNX_GENAI_RUNTIME: + + if util.find_spec("onnxruntime_genai"): + + try: + global og + og = importlib.import_module("onnxruntime_genai") + GLOBAL_ONNX_GENAI_RUNTIME = True + except: + raise LLMWareException(message="ONNXQNNGenerativeModel: could not load onnxruntime_genai module. " + "To fix: please check the following:\n" + "1. pip install onnxruntime_qnn==1.22.2\n" + "2. pip install onnxruntime_genai==0.9.0\n" + "3. confirm Windows Arm64 with Snapdragon NPU") + + # sets trailing space default when constructing the prompt + # in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves + # performance + + self.trailing_space = "" + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + + self.model_type = None + self.config = None + + # parameters on context len + output generation + self.max_total_len = context_window + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + + # key output parameters + self.max_output = max_output + self.target_requested_output_tokens = self.max_output + + self.model_architecture = None + self.separator = "\n" + + # use 0 as eos token id by default in generation -> but try to pull from model config + self.eos_token_id = 0 + + self.use_gpu = False + + # coming soon + self.windows_local_foundry_active = False + + # no api key expected or required + self.api_key = api_key + + self.error_message = "\nUnable to identify and load HuggingFace model." + + # temperature settings + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at default of 0.3 + self.temperature = 0.3 + + self.add_prompt_engineering = False + self.add_context = "" + self.context = "" + self.prompt = "" + + # not currently implemented for this model class + self.api_endpoint = api_endpoint + + self.model_repo_path = None + + # confirm platform check + import sys + import platform + plat = sys.platform + mach = platform.machine().lower() + logger.info(f"ONNXQNNGenerativeModel - platform - {plat} - machine - {mach}") + + if not (plat == "win32" and mach == "arm64"): + logger.warning(f"ONNXQNNGenerativeModel is designed for Windows Arm64.") + + self.post_init() + + def load_model_for_inference(self, loading_directions, model_card=None): + + """ Loads ONNX Model from local path using loading directions. """ + + self.model_repo_path = loading_directions + + if model_card: + self.model_card = model_card + + self.validate() + + onnx_model_path = os.path.join(LLMWareConfig().get_model_repo_path(), + self.model_name) + + if self.npu_optimized: + # get npu optimized onnxruntime with qnn + set_for_npu_qnn = True + + # starting with onnxruntime-qnn 2.0, need to set qnn execution provider path + # e.g., path to "onnxruntime_providers_qnn.dll" + + qnn_path = os.environ.get("qnn_onnx_path","") + if not qnn_path: + # by default, look in the onnxruntime_qnn package + import onnxruntime_qnn + backend_path = os.path.dirname(onnxruntime_qnn.__file__) + qnn_path = os.path.join(backend_path, "onnxruntime_providers_qnn.dll") + + # register the backend + og.register_execution_provider_library("QNNExecutionProvider", qnn_path) + + logger.info(f"ONNXQNNGenerativeModel - load_model_for_inference - qnn path - {qnn_path}") + + # use global onnxruntime_genai - constructing model from config + config = og.Config(onnx_model_path) + self.model = og.Model(config) + + self.tokenizer = og.Tokenizer(self.model) + self.tokenizer_stream = self.tokenizer.create_stream() + + search_options = {} + search_options['max_length'] = 2048 + search_options['batch_size'] = 1 + self.params = og.GeneratorParams(self.model) + self.params.set_search_options(**search_options) + + logger.info(f"ONNXQNNGenerativeModel - constructed model - {self.model_name}.") + + return self + + def unload_model(self): + """ Not implemented. """ + return True + + def set_api_key(self, api_key, env_var=""): + """ Not implemented for ONNXQNNGenerativeModel """ + return True + + def _get_api_key(self, env_var=""): + """ Not implemented for ONNXQNNGenerativeModel """ + return True + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None): + + """ Executes generation inference on model. """ + + # first prepare the prompt + t0 = time.time() + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + text_prompt = prompt_final + self.trailing_space + + input_tokens = self.tokenizer.encode(text_prompt) + + token_count = 0 + output = "" + + generator = og.Generator(self.model, self.params) + + # note: onnxruntime_genai library makes a lot of small breaking changes + # in their generation loops -> this should be OK with versions >0.9.0 + # if you see error, then check the documentation for onnxruntime_genai + # which is pretty good at explaining/documenting the change and how to fix + + generator.append_tokens(input_tokens) + + try: + + while not generator.is_done(): + + token_count += 1 + + # change in v0.6 api - explicit compute logits call not required + # generator.compute_logits() + + generator.generate_next_token() + + # not activated currently + self.get_logits = False + # to get logit value + if self.get_logits: + logit = generator.get_output("logits") + self.register_top_logits(logit) + + new_token = generator.get_next_tokens()[0] + + if self.get_logits: + self.output_tokens.append(new_token) + + output += self.tokenizer_stream.decode(new_token) + + if token_count > self.max_output: + break + + except Exception as e: + logger.warning(f"ONNXQNNGenerativeModel inference produced error - {e}") + pass + + del generator + + usage = {"input": len(input_tokens), + "output": token_count, + "total": len(input_tokens) + token_count, + "metric": "tokens", + "processing_time": time.time() - t0} + + output_response = {"llm_response": output, "usage": usage} + + if self.get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None, skip_pe_override=False): + + """ Executes stream generation inference on model. """ + + # first prepare the prompt + t0 = time.time() + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + self.preview() + + text_prompt = self.prompt + + if self.add_prompt_engineering and not skip_pe_override: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + text_prompt = prompt_final + self.trailing_space + + logger.debug("ONNXQNNGenerative Model - onnx stream starting.") + + input_tokens = self.tokenizer.encode(text_prompt) + + token_count = 0 + output = "" + + # note: onnxruntime_genai library makes a lot of small breaking changes + # in their generation loops -> this should be OK with versions > 0.9.0 + # if you see error, then check the documentation for onnxruntime_genai + # which is pretty good at explaining/documenting the change and how to fix + + self.generator = og.Generator(self.model, self.params) + + self.generator.append_tokens(input_tokens) + + while True: + + token_count += 1 + + # change in v0.6 api - no explicit compute logits call + # self.generator.compute_logits() + + self.generator.generate_next_token() + + if self.generator.is_done(): + break + + self.get_logits = False + # to get logit value + if self.get_logits: + logit = self.generator.get_output("logits") + self.register_top_logits(logit) + + new_token = self.generator.get_next_tokens()[0] + + if self.get_logits: + self.output_tokens.append(new_token) + + output += self.tokenizer_stream.decode(new_token) + + if token_count > self.max_output: + break + + yield self.tokenizer_stream.decode(new_token) + + self.generator = None + + usage = {"input": len(input_tokens), + "output": token_count, + "total": len(input_tokens) + token_count, + "metric": "tokens", + "processing_time": time.time() - t0} + + output_response = {"llm_response": output, "usage": usage} + + if self.get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + logger.debug("ONNXQNNGenerativeModel - completed stream generation.") + + return output_response + + def cleanup_stream_gen_on_early_stop(self): + + self.generator = None + return True + + def register_top_logits(self, logit): + + """ Gets the top logits and keeps a running log for output analysis. """ + + # logit will be in form of (1,1,vocab_len), for all but the first logit + # if first logit (will have shape of context len - add [-1]) + + if logit.shape[1] > 1: + # used for first logit with shape, e.g., (1,input_token_len,vocab_size) + logit_array = logit.squeeze()[-1] + else: + # all other logits after the first token + logit_array = logit.squeeze() + + logit_size = logit.shape[-1] + + # useful check on shape of logit_array + logit_array_size = logit_array.shape + + sm = np.exp(logit_array) / sum(np.exp(logit_array)) + + sm_sorted = np.sort(sm) + sm_args_sorted = np.argsort(sm) + + top_logits = [] + + for x in range(0, self.top_logit_count): + # round the float number to 3 digits + pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3)) + top_logits.append(pair) + + self.logits_record.append(top_logits) + + return top_logits + + +class LLMWareModel(BaseModel): + + """LLMWareModel class implements the API for LLMWare generative models. """ + + def __init__(self, model_name=None, api_key=None, context_window=2048, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "LLMWareModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + self.api_key = api_key + + self.model_name = model_name + self.model = None + self.tokenizer = None + + self.error_message = "\nUnable to connect to LLMWare GPT API. Please try again later." + + # set max_total_len -> adjust input and output based on use case + self.max_total_len = context_window + self.max_input_len = int(0.4 * context_window) + self.llm_max_output_len = int(0.4 * context_window) + + self.separator = "\n" + + # inference settings + self.temperature = 0.2 + self.target_requested_output_tokens = 200 + self.add_prompt_engineering = True + self.add_context = "" + self.prompt = "" + + self.post_init() + + def set_api_key(self, api_key, env_var="USER_MANAGED_LLMWARE_GPT_API_KEY"): + + """ Utility method to set the API key in os.environ variable. """ + + os.environ[env_var] = api_key + logger.info("update: added and stored READ_GPT api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_LLMWARE_GPT_API_KEY"): + + """ Utility method to get api_key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Gets GPT2 tokenizer for fast approximate token counting. """ + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + return len(toks) + + def prompt_engineer(self, query, context, inference_dict=None): + + """ Builds prompt by assembling query, context and applying the selected prompt style. """ + + if not query: + query = "What is a list that summarizes the key points?" + + # default_case + prompt_engineered = context + "\n" + query + + if self.add_prompt_engineering == "top_level_summary_select": + prompt_engineered += query + "\n" + prompt_engineered += "Which of the following selections best answers the question?" + prompt_engineered += context + + if self.add_prompt_engineering == "summarize_with_bullets_no_query": + issue = "What is a list of the most important points?" + prompt_engineered = context + "\n" + issue + + return prompt_engineered + + def load_model_for_inference(self, model_name=None, model_card=None,fp=None, **kwargs): + + # validate before loading - turned off + # self.validate() + + # look up model_name in configs + if model_name: + self.model_name = model_name + return self + + def load_pretrained_model(self, model_name=None): + if model_name: + self.model_name = model_name + # convenience method for pretrained models as a single step + return self + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes inference on LLMWare Model. Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # call to preview hook (not implemented by default) + self.preview() + + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + + # safety check on length - set cap with small 'buffer' + input_tokens = self.token_counter(prompt_enriched) + buffer = 10 + available_tokens_in_output_context_window = self.max_total_len - input_tokens - buffer + # if target requested output is less, then keep - otherwise, cap with 'safe' maximum len + target_len = min(self.target_requested_output_tokens, available_tokens_in_output_context_window) + + output_dict_new = {} + output_response = {} + usage = {"input": input_tokens} + + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + params = {"context": self.add_context, + "question": self.prompt, + "max_output_tokens": target_len, + "api_key": self.api_key} + + # params = {"context": prompt["context"],"question": prompt["query"], "max_output_tokens": 50, "api_key": good_key} + + time_start = time.time() + + try: + + output = requests.post(os.environ.get("LLMWARE_GPT_URI"), data=params) + output_dict_new = ast.literal_eval(output.text) + success_path = 1 + output_response = output_dict_new + + except: + + text_output = "/***ERROR***/" + usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens", + "processing_time": time.time() - time_start} + + logger.error("error: no response from aib remote server for llmware-gpt model - " + "check api key and connection") + + success_path = -1 + output_response = {"llm_response": "", "usage": usage} + + # output inference parameters + self.llm_response = "" + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class OpenAIEmbeddingModel(BaseModel): + + """ OpenAIEmbeddingModel class implements the OpenAI API for embedding models. """ + + def __init__(self, model_name=None, api_key=None, embedding_dims=None, model_card=None, max_len=None, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "OpenAIEmbeddingModel" + self.model_category = "embedding" + + # must have elements for embedding model + self.model_name = model_name + self.api_key = api_key + self.model_card = model_card + self.tokenizer = None + + if not embedding_dims: + self.embedding_dims = 1536 + else: + self.embedding_dims = embedding_dims + + # openai standard for embeddings is 8191 as of feb 2024 + self.max_total_len = 8191 + self.max_len = self.max_total_len + + if model_card: + if "embedding_dims" in model_card: + self.embedding_dims = model_card["embedding_dims"] + + if "context_window" in model_card: + self.max_total_len = model_card["context_window"] + + self.error_message = "\nUnable to connect to OpenAI. Please try again later." + + if max_len: + if max_len < self.max_total_len: + self.max_len = max_len + + self.text_sample = None + + self.post_init() + + def set_api_key(self, api_key,env_var="USER_MANAGED_OPENAI_API_KEY"): + + """ Utility method to set the API key in os.environ variable. """ + + os.environ[env_var] = api_key + logger.info("update: added and stored OpenAI api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_OPENAI_API_KEY"): + + """ Utility method to get api_key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + return self.api_key + + def get_tokenizer(self): + self.tokenizer = Utilities().get_default_tokenizer() + return self.tokenizer + + def token_counter(self, text_sample): + + """ Counts tokens in text sample. """ + + return len(self.tokenizer.encode(text_sample).ids) + + def embedding(self, text_sample, api_key=None): + + self.text_sample = text_sample + + # call to preview (not implemented by default) + self.preview() + + if api_key: + self.api_key = api_key + + if not self.api_key: + self.api_key = self._get_api_key() + + if not self.api_key: + logger.error("error: invoking OpenAI Embedding model with no api_key") + + # need to prepare for batches + if isinstance(self.text_sample, list): + text_prompt = self.text_sample + input_len = len(text_sample) + else: + text_prompt = [self.text_sample] + input_len = 1 + + try: + from openai import OpenAI + except ImportError: + raise DependencyNotInstalledException("openai >= 1.0") + + from llmware.configs import OpenAIConfig + + # insert safety check here + safe_samples = [] + safety_buffer = 200 + if self.max_total_len < 8191: + self.max_total_len = 8191 + + tokenizer = self.get_tokenizer() + + for sample in text_prompt: + + tok_len = self.token_counter(sample) + + if tok_len < (self.max_total_len - safety_buffer): + safe_samples.append(sample) + + else: + + if len(sample) > 300: + display_sample = sample[0:300] + " ... " + else: + display_sample = sample + + logger.warning(f"warning: OpenAI Embedding - input sample len - {tok_len} > context_window size " + f"\ninput_sample - {display_sample} " + f"\n\nSample is being truncated.") + + tok = tokenizer.encode(sample).ids + tok = tok[0:(self.max_total_len - safety_buffer)] + sample = tokenizer.decode(tok) + safe_samples.append(sample) + + text_prompt = safe_samples + # end - safety check + + # update to open >v1.0 api - create client and output is pydantic objects + + azure_client = OpenAIConfig().get_azure_client() + + if not azure_client: + client = OpenAI(api_key=self.api_key) + + else: + + logger.info("update: applying custom OpenAI client from OpenAIConfig") + + client = azure_client + + response = client.embeddings.create(model=self.model_name, input=text_prompt) + + if input_len == 1: + embedding = response.data[0].embedding + else: + embedding = [] + for i, entries in enumerate(response.data): + embedding.append(response.data[i].embedding) + + self.register() + + return embedding + + +class HFReRankerModel(BaseModel): + + """HFReRankerModel class implements the interface for HuggingFace ReRanker models. """ + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None, + embedding_dims=None, trust_remote_code=False, use_gpu_if_available=True, max_len=None, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "HFReRankerModel" + self.model_category = "reranker" + + # pull in expected hf input + self.model_name = model_name + self.model = model + self.tokenizer= tokenizer + self.embedding_dims = embedding_dims + self.model_type = None + self.max_total_len = 2048 + self.model_architecture = None + self.model_card = model_card + self.safe_buffer = 12 + + # default for HF embedding model -> will be over-ridden by model card / configs, if available + self.context_window = 512 + + if self.model_card: + if "embedding_dims" in self.model_card: + self.embedding_dims = self.model_card["embedding_dims"] + + if "context_window" in self.model_card: + self.context_window = self.model_card["context_window"] + + # insert dynamic pytorch load here + global GLOBAL_TORCH_IMPORT + if not GLOBAL_TORCH_IMPORT: + + logger.debug("update: ModelCatalog - HFReRankerModel - local dynamic load of torch here") + if util.find_spec("torch"): + + try: + global torch + torch = importlib.import_module("torch") + GLOBAL_TORCH_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load torch module.") + + else: + raise LLMWareException(message="Exception: need to import torch to use this class.") + + # end dynamic import here + + if self.model_name and not model: + + # pull from HF + hf_repo_name = self.model_name + + if not self.model_card: + self.model_card = ModelCatalog().lookup_model_card(model_name) + + if self.model_card: + if "hf_repo" in self.model_card: + hf_repo_name = self.model_card["hf_repo"] + + pt_loader = PyTorchLoader(api_key=api_key,trust_remote_code=trust_remote_code,custom_loader=None) + + self.model=pt_loader.get_reranker_model(hf_repo_name) + self.tokenizer=None + + self.use_gpu = torch.cuda.is_available() and use_gpu_if_available + + if self.model: + + self.config = self.model.config.to_dict() + + if "hidden_size" in self.config: + self.embedding_dims = self.config["hidden_size"] + + if "model_type" in self.config: + self.model_type = self.config["model_type"] + + if "max_position_embeddings" in self.config: + + try: + self.context_window = int(self.config["max_position_embeddings"]) + except: + pass + + if "_name_or_path" in self.config: + self.model_name = self.config["_name_or_path"] + + if "architectures" in self.config: + if isinstance(self.config["architectures"],list): + self.model_architectures = self.config["architectures"][0] + else: + self.model_architectures = self.config["architectures"] + + self.model.eval() + + if self.use_gpu: + self.model.to('cuda') + + else: + raise ModelNotFoundException(model_name) + + # no api key expected or required + self.api_key = api_key + + # set max len for tokenizer truncation with 'safe_buffer' below context_window size + if self.context_window > self.safe_buffer: + self.max_len = self.context_window - self.safe_buffer + else: + self.max_len = self.context_window + + # option to set smaller size than model context window + if max_len: + if max_len < self.context_window: + self.max_len = max_len + + self.query = "" + self.text_results = None + + self.post_init() + + def set_api_key(self, api_key, env_var="USER_MANAGED_HF_API_KEY"): + + """ Sets the API key - generally not needed for public HF repositories. """ + + os.environ[env_var] = api_key + logger.info("update: added and stored HF api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_HF_API_KEY"): + + """ Gets API key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("error: _get_api_key could not successfully retrieve value from: %s ", env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Counts tokens in text sample. Not currently implemented. """ + + return -1 + + def inference (self, query, text_results, api_key=None, top_n=20, relevance_threshold=None, min_return=3): + + """ Executes reranking inference. """ + + self.query = query + self.text_results = text_results + + # call to preview (not implemented by default) + self.preview() + + documents = [] + for i, chunks in enumerate(text_results): + documents.append(chunks['text']) + + sentence_pairs = [[self.query, doc] for doc in documents] + + scores = self.model.compute_score(sentence_pairs) + + output = [] + for i, score in enumerate(scores): + text_results[i].update({"rerank_score": score}) + output.append(text_results[i]) + + ranked_output = sorted(output, key=lambda x: x["rerank_score"], reverse=True) + + # will return top_n if no relevance threshold set + if not relevance_threshold: + if top_n < len(ranked_output): + final_output = ranked_output[0:top_n] + else: + final_output = ranked_output + else: + final_output = [] + # if relevance threshold, will return all results above threshold + for entries in ranked_output: + if entries["rerank_score"] >= relevance_threshold: + final_output.append(entries) + + # fallback, if no result above threshold, then will return the min number of results + if len(final_output) == 0: + final_output = ranked_output[0:min_return] + + self.register() + + return final_output + + +class HFEmbeddingModel(BaseModel): + + """HFEmbeddingModel class implements the API for HuggingFace embedding models. """ + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None, + embedding_dims=None, trust_remote_code=False, use_gpu_if_available=True, max_len=None, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "HFEmbeddingModel" + self.model_category = "embedding" + + # pull in expected hf input + self.model_name = model_name + self.model = model + self.tokenizer= tokenizer + self.embedding_dims = embedding_dims + self.model_type = None + self.max_total_len = 2048 + self.model_architecture = None + self.model_card = model_card + self.safe_buffer = 12 + + # default for HF embedding model -> will be over-ridden by model card / configs, if available + self.context_window = 512 + + if self.model_card: + if "embedding_dims" in self.model_card: + self.embedding_dims = self.model_card["embedding_dims"] + + if "context_window" in self.model_card: + self.context_window = self.model_card["context_window"] + + # insert dynamic pytorch load here + global GLOBAL_TORCH_IMPORT + if not GLOBAL_TORCH_IMPORT: + + logger.debug("update: ModelCatalog - HFEmbeddingModel - local dynamic load of torch here") + if util.find_spec("torch"): + + try: + global torch + torch = importlib.import_module("torch") + GLOBAL_TORCH_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load torch module.") + + else: + raise LLMWareException(message="Exception: need to import torch to use this class.") + + # end dynamic import here + + if self.model_name and not model: + + # pull from HF + hf_repo_name = self.model_name + + if not self.model_card: + self.model_card = ModelCatalog().lookup_model_card(model_name) + + if self.model_card: + if "hf_repo" in self.model_card: + hf_repo_name = self.model_card["hf_repo"] + + pt_loader = PyTorchLoader(api_key=api_key,trust_remote_code=trust_remote_code,custom_loader=None) + + self.model=pt_loader.get_embedding_model(hf_repo_name) + self.tokenizer=pt_loader.get_tokenizer(hf_repo_name) + + self.use_gpu = torch.cuda.is_available() and use_gpu_if_available + + if self.model: + + self.config = self.model.config.to_dict() + + if "hidden_size" in self.config: + self.embedding_dims = self.config["hidden_size"] + + if "model_type" in self.config: + self.model_type = self.config["model_type"] + + if "max_position_embeddings" in self.config: + + try: + self.context_window = int(self.config["max_position_embeddings"]) + except: + pass + + if "_name_or_path" in self.config: + self.model_name = self.config["_name_or_path"] + + if "architectures" in self.config: + if isinstance(self.config["architectures"],list): + self.model_architectures = self.config["architectures"][0] + else: + self.model_architectures = self.config["architectures"] + + self.model.eval() + + if self.use_gpu: + self.model.to('cuda') + + else: + raise ModelNotFoundException(model_name) + + # no api key expected or required + self.api_key = api_key + + # set max len for tokenizer truncation with 'safe_buffer' below context_window size + if self.context_window > self.safe_buffer: + self.max_len = self.context_window - self.safe_buffer + else: + self.max_len = self.context_window + + # option to set smaller size than model context window + if max_len: + if max_len < self.context_window: + self.max_len = max_len + + self.text_sample = None + + self.post_init() + + def set_api_key(self, api_key, env_var="USER_MANAGED_HF_API_KEY"): + + """ Sets the API key - generally not needed for public HF repositories. """ + + os.environ[env_var] = api_key + logger.info("update: added and stored HF api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_HF_API_KEY"): + + """ Gets API key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("error: _get_api_key could not successfully retrieve value from: %s ", env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Counts tokens in text sample. """ + + # need to support HF tokenizer + toks = self.tokenizer.encode(text_sample).ids + return len(toks) + + def embedding (self, text_sample, api_key=None): + + """ Executes embedding inference. """ + + self.text_sample = text_sample + + # call to preview (not implemented by default) + self.preview() + + # return embeddings only + if isinstance(self.text_sample,list): + sequence = self.text_sample + + else: + sequence = [self.text_sample] + + model_inputs = self.tokenizer(sequence, truncation=True, max_length=self.max_len, return_tensors="pt",padding=True) + + if self.use_gpu: + input_ids = model_inputs.input_ids.to('cuda') + attn_mask = model_inputs.attention_mask.to('cuda') + else: + input_ids = model_inputs.input_ids.to('cpu') + attn_mask = model_inputs.attention_mask.to('cpu') + + # context manager to run inference without saving/calculating grads + with torch.no_grad(): + model_outputs = self.model(input_ids, attention_mask=attn_mask) + + embedding = model_outputs.last_hidden_state[:,0] + + # normalize hf embeddings + embeddings_normalized = torch.nn.functional.normalize(embedding, p=2, dim=1) + + if self.use_gpu: + embeddings_normalized = np.array(embeddings_normalized.detach().to('cpu')) + else: + embeddings_normalized = embeddings_normalized.detach().numpy() + + self.register() + + return embeddings_normalized + + +class HFGenerativeModel(BaseModel): + + """ HFGenerativeModel class implements the HuggingFace generative model API, and is used generally for + models in HuggingFace repositories, e.g., Dragon, Bling, etc. """ + + # support instantiating HF model in two different ways: + # 1. directly passing a previously loaded HF model object and tokenizer object + # 2. passing a model_name only, which will then create the model and tokenizer + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None, + prompt_wrapper=None, instruction_following=False, context_window=2048, + use_gpu_if_available=True, trust_remote_code=True, sample=True,max_output=100, temperature=0.3, + get_logits=False, api_endpoint=None, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "HFGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + # pull in expected hf input + self.model_name = model_name + self.hf_tokenizer_name = model_name + self.model = model + self.tokenizer = tokenizer + + # new parameters + self.sample=sample + self.get_logits=get_logits + self.auto_remediate_function_call_output = True + + # Function Call parameters + self.model_card = model_card + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.primary_keys = None + self.function = None + self.fc_supported = False + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + # insert dynamic pytorch load here + if not api_endpoint: + + global GLOBAL_TORCH_IMPORT + if not GLOBAL_TORCH_IMPORT: + if util.find_spec("torch"): + + try: + global torch + torch = importlib.import_module("torch") + GLOBAL_TORCH_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load torch module.") + + else: + raise LLMWareException(message="Exception: need to import torch to use this class.") + + # end dynamic import here + + # instantiate if model_name passed without actual model and tokenizer + if model_name and not model and not tokenizer and not api_endpoint: + + hf_repo_name = self.model_name + + if not self.model_card: + self.model_card = ModelCatalog().lookup_model_card(self.model_name) + + if self.model_card: + if "hf_repo" in self.model_card: + hf_repo_name = self.model_card["hf_repo"] + self.hf_tokenizer_name = hf_repo_name + + pt_loader = PyTorchLoader(api_key=api_key, trust_remote_code=trust_remote_code, custom_loader=None) + self.model = pt_loader.get_generative_model(hf_repo_name) + self.tokenizer = pt_loader.get_tokenizer(hf_repo_name) + + # set to defaults for HF models in Model Catalog + # this can be over-ridden post initiation if needed for custom models + self.prompt_wrapper = "human_bot" + self.instruction_following = False + + # set specific parameters associated with custom models + # note - these two parameters will control how prompts are handled - model-specific + self.prompt_wrapper = prompt_wrapper + self.instruction_following = instruction_following + + if not model_card: + # safety - empty iterable rather than 'None' + model_card = [] + + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + else: + self.instruction_following = False + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + # sets trailing space default when constructing the prompt + # in most cases, this is * no trailing space * but for some models, a trailing space or "\n" improves + # performance + + self.trailing_space = "" + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + + self.model_type = None + self.config = None + + # parameters on context len + output generation + self.max_total_len = context_window + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + + # key output parameters + self.max_output=max_output + self.target_requested_output_tokens = self.max_output + + self.model_architecture = None + self.separator = "\n" + + # use 0 as eos token id by default in generation -> but try to pull from model config + self.eos_token_id = 0 + + # will load model and inference onto gpu, + # if (a) CUDA available and (b) use_gpu_if_available set to True (default) + if not api_endpoint: + self.use_gpu = torch.cuda.is_available() and use_gpu_if_available + else: + self.use_gpu = False + + if self.model: + + if isinstance(self.model.config, dict): + self.config = self.model.config + else: + self.config = self.model.config.to_dict() + + if "trailing_space" in self.config: + self.trailing_space = self.config["trailing_space"] + + if "eos_token_id" in self.config: + # only use to set if value is not None + if self.config["eos_token_id"]: + self.eos_token_id = self.config["eos_token_id"] + + if "model_type" in self.config: + self.model_type = self.config["model_type"] + + if "hidden_size" in self.config: + self.embedding_dims = self.config["hidden_size"] + + if "max_position_embeddings" in self.config: + self.max_total_len = self.config["max_position_embeddings"] + + if "architectures" in self.config: + if isinstance(self.config["architectures"], list): + self.model_architectures = self.config["architectures"][0] + else: + self.model_architectures = self.config["architectures"] + + # prepare model for inference + self.model.eval() + + if self.use_gpu: + self.model.to('cuda') + logger.debug("update: HFGenerative loading - moving model to cuda") + + else: + if not api_endpoint: + logger.error("error: HFGenerativeModel - could not identify model - ", model_name) + + # no api key expected or required + self.api_key = api_key + + self.error_message = "\nUnable to identify and load HuggingFace model." + + # temperature settings + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at default of 0.3 + self.temperature = 0.3 + + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + self.context = "" + self.tool_type = None + + self.api_endpoint = api_endpoint + + self.post_init() + + def set_api_key(self, api_key, env_var="USER_MANAGED_HF_API_KEY"): + + """ Sets the API key - generally not needed for public HF repositories. """ + + os.environ[env_var] = api_key + logger.info("update: added and stored HF api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_HF_API_KEY"): + + """ Gets API key from os.environ variable. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("error: _get_api_key could not successfully retrieve value from: %s ", env_var) + + return self.api_key + + def token_counter(self, text_sample): + + """ Quick approximate token counter - uses default tokenizer so may have minor differences from the + model's actual tokenization. """ + + tokenizer = Utilities().get_default_tokenizer() + toks = tokenizer.encode(text_sample).ids + + return len(toks) + + def prompt_engineer(self, query, context, inference_dict): + + """ Applies prompt and templating preparation. """ + + # if loaded model was not pretrained on instruction_following, then skip any instructions + if not self.instruction_following: + + if context: + output = context + "\n" + query + else: + output = query + + # unlikely that there would be an 'instruct wrapping' on text, but allow for possibility + if self.prompt_wrapper: + output = PromptCatalog().apply_prompt_wrapper(output, self.prompt_wrapper, + instruction=None) + + return output + + # move ahead to add instructions and prompt engineering + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + selected_prompt = self.add_prompt_engineering + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, + context=context, + inference_dict=inference_dict) + + if prompt_dict: + prompt_engineered = prompt_dict["core_prompt"] + else: + # default case + prompt_engineered = "Please read the following text: " + context + self.separator + prompt_engineered += "Based on this text, please answer the question: " + query + self.separator + prompt_engineered += "Please answer the question only with facts provided in the materials. " \ + "If the question can not be answered in the materials, then please " \ + "respond 'Not Found.'" + + # final wrapping, based on model-specific instruct training format + # --provides a final 'wrapper' around the core prompt text, based on model expectations + + if self.prompt_wrapper: + prompt_engineered = PromptCatalog().apply_prompt_wrapper(prompt_engineered, self.prompt_wrapper, + instruction=None) + + return prompt_engineered + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None): + + """ Executes generation inference on model. """ + + self.prompt = prompt + + # first prepare the prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # add defaults if add_prompt_engineering not set + if not self.add_prompt_engineering: + + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - defaults update + + # show warning if function calling model + if self.fc_supported: + logger.warning("This is a function calling model - using .inference may lead to unexpected " + "results. Recommended to use the .function_call method to ensure correct prompt " + "template packaging.") + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # call to preview (not implemented by default) + self.preview() + + # START - route to api endpoint + if self.api_endpoint: + return self.inference_over_api_endpoint(self.prompt, context=self.add_context, + inference_dict=inference_dict) + # END - route to api endpoint + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # text_prompt = prompt_final + "\n" + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + text_prompt = prompt_final + self.trailing_space + + # second - tokenize to get the input_ids + + tokenizer_output = self.tokenizer.encode(text_prompt) + input_token_len = len(tokenizer_output) + input_ids = torch.tensor(tokenizer_output).unsqueeze(0) + + # explicit check and setting to facilitate debugging + if self.use_gpu: + input_ids = input_ids.to('cuda') + else: + input_ids = input_ids.to('cpu') + + # time start + time_start = time.time() + + # This simplified greedy sampling generation loop was derived from and inspired by ideas in the + # HuggingFace transformers library generation class. + # https: //github.com/huggingface/transformers/tree/main/src/transformers/generation + # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc.team, and NVIDIA Corporation. + # Licensed under the Apache License, Version 2.0 (the "License") + + # default settings + pad_token_id = 0 + + # for most models, eos_token_id = 0, but llama and mistral = 2 + eos_token_id = [self.eos_token_id] + # eos_token_id = [0] + + eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) + + # keep track of which sequences are already finished + unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device) + + this_peer_finished = False # used by synced_gpus only + # auto-regressive generation + new_tokens_generated = 0 + + attn_mask = torch.ones(input_ids.shape[1]).unsqueeze(0) + + # explicit check and setting to facilitate debugging, if needed + if self.use_gpu: + attn_mask = attn_mask.to('cuda') + else: + attn_mask = attn_mask.to('cpu') + + batch_size = input_ids.shape[0] + seq_len = input_ids.shape[1] + + pkv = None + + # borrow setting from GGUFConfigs + get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed") + t_gen_start = time.time() + first_token_processing_time = -1.0 + + while True: + + inp_one_time: torch.LongTensor = input_ids + + if new_tokens_generated > 0: + inp_one_time = input_ids[:, -1:] + + # explicit check and setting to facilitate debugging, if needed + if self.use_gpu: + inp0 = inp_one_time.to('cuda') + inp1 = attn_mask.to('cuda') + else: + inp0 = inp_one_time.to('cpu') + inp1 = attn_mask.to('cpu') + + # inp3 = torch.LongTensor([new_tokens_generated]) + + # need to invoke forward pass on model + # outputs = self.model(inp0,inp1,pkv) + + # context manager to avoid saving/computing grads in forward pass + with torch.no_grad(): + outputs = self.model(input_ids=inp0, attention_mask=inp1, past_key_values=pkv, + return_dict=True) + + if new_tokens_generated == 0: + if get_first_token_speed: + first_token_processing_time = time.time() - t_gen_start + + new_tokens_generated += 1 + + next_token_logits = outputs.logits[:, -1, :] + + # capture top logits - not currently activated for inference + # self.register_top_logits(next_token_logits) + # shape of next_token_logits = torch.Size([1, 32000]) + # logger.debug(f"next token logits shape - {next_token_logits.shape}") + + if self.temperature and self.sample: + next_token_scores = next_token_logits / self.temperature + else: + next_token_scores = next_token_logits + + # get token from logits + probs = torch.nn.functional.softmax(next_token_scores, dim=-1) + + if not self.sample: + # will pull the 'top logit' only + next_tokens = torch.argmax(probs).unsqueeze(0) + else: + # will apply probabilistic sampling + next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) + + # new - option to capture logits and output tokens for analysis + if self.get_logits: + self.register_top_logits(next_token_logits) + + # capture the output tokens + if self.use_gpu: + next_tokens_np = np.array(next_tokens.to('cpu')) + else: + + next_tokens_np = np.array(next_tokens) + + self.output_tokens.append(next_tokens_np[0]) + + # finished sentences should have their next token be a padding token + if eos_token_id is not None: + next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences) + + # update generated ids, model inputs, and length for next step + input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + + # testing output in progress starts here + """ + logger.debug(f"update: input_ids - {input_ids}") + # outputs_detached = outputs.to('cpu') + outputs_np = np.array(input_ids[0]) + output_str = self.tokenizer.decode(outputs_np) + logger.debug(f"update: output string - {output_str}") + """ + # end - testing output in progress + + pkv = outputs.past_key_values + + # update attention mask + attn_mask = torch.cat([attn_mask, attn_mask.new_ones((attn_mask.shape[0], 1))], dim=-1) + + # if eos_token was found in one sentence, set sentence to finished + if eos_token_id_tensor is not None: + unfinished_sequences = unfinished_sequences.mul( + next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod(dim=0) + ) + + # stop when each sentence is finished + if unfinished_sequences.max() == 0: + this_peer_finished = True + + # stop if we exceed the maximum length + if new_tokens_generated >= self.target_requested_output_tokens: + this_peer_finished = True + + if this_peer_finished: + break + + # Generation completed - prepare the output + + if self.use_gpu: + outputs_np = np.array(input_ids[0].to('cpu')) + else: + outputs_np = np.array(input_ids[0]) + + output_only = outputs_np[input_token_len:] + + output_str = self.tokenizer.decode(output_only) + + # post-processing clean-up - stop at endoftext + eot = output_str.find("<|endoftext|>") + if eot > -1: + output_str = output_str[:eot] + + # new post-processing clean-up - stop at + eots = output_str.find("") + if eots > -1: + output_str = output_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = output_str.find(":") + if bot > -1: + output_str = output_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = output_str.find("") + if boss > -1: + output_str = output_str[boss + len(""):] + + # end - post-processing + + total_len = len(outputs_np) + + usage = {"input": input_token_len, + "output": total_len - input_token_len, + "total": total_len, + "metric": "tokens", + "processing_time": time.time() - time_start} + + if get_first_token_speed: + usage.update({"first_token_processing_time": first_token_processing_time}) + + output_response = {"llm_response": output_str, "usage": usage} + + if self.get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output_str + self.usage = usage + self.final_prompt = text_prompt + + self.register() + + return output_response + + def fc_prompt_engineer(self, context, params=None, function=None): + + """ Prompt engineering for Function Call prompts. """ + + if not params: + params = self.primary_keys + + # add safety check in looking for default self.function pulled from model card + if not function: + if self.function: + if isinstance(self.function,list): + if len(self.function) > 0: + function = self.function[0] + else: + function = self.function + + # if not successful identifying a function, then choose 'classify' by default + if not function: + function = "classify" + + # prepare SLIM prompt + class_str = "" + for key in params: + class_str += str(key) + ", " + if class_str.endswith(", "): + class_str = class_str[:-2] + + f = str(function) + + # key templating format for SLIM function calls + full_prompt = ": " + context + "\n" + "<{}> {} ".format(f, class_str, f) + "\n:" + + full_prompt = full_prompt + self.trailing_space + + return full_prompt + + def register_top_logits(self, next_token_logit): + + """ Retrieves the logits for current sample, and packages into indexed top list and + registers in self.logit_record. """ + + # assumes input of next_token_logit from generation script + # will be a tensor of shape [1,vocab_size] + + logit_size = next_token_logit.shape[-1] + logit = torch.squeeze(next_token_logit) + + if self.use_gpu: + logit_array = np.array(logit.to('cpu')) + else: + logit_array = np.array(logit) + + sm = np.exp(logit_array) / sum(np.exp(logit_array)) + + sm_sorted = np.sort(sm) + sm_args_sorted = np.argsort(sm) + + top_logits = [] + # by default, self.top_logit_count = 10, will get the top 10 highest values in logit output + for x in range(0, self.top_logit_count): + # experiment - rounding the long float number + pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1],3)) + top_logits.append(pair) + + self.logits_record.append(top_logits) + + return top_logits + + def function_call(self, context, function=None, params=None, get_logits=True, + temperature=-99, max_output=None): + + """ This is the key inference method for SLIM models - takes a context passage and a key list + which is packaged in the prompt as the keys for the dictionary output""" + + self.context = context + + # only assign self.function if a function has been passed in the call + if function: + self.function = function + + if not self.fc_supported: + logger.warning("HFGenerativeModel - loaded model does not support function calls. " + "Please either use the standard .inference method with this model, or use a " + "model that has 'function_calls' key set to True in its model card.") + return [] + + # reset and start from scratch with new function call + self.output_tokens = [] + self.logits_record = [] + + if temperature != -99: + self.temperature = temperature + + if max_output: + self.target_requested_output_tokens = max_output + + if get_logits: + self.get_logits = get_logits + + if params: + self.primary_keys = params + + # call to preview (not implemented by default) + self.preview() + + if not self.primary_keys: + logger.warning("warning: function call - no keys provided - function call may yield unpredictable results") + + # START - route to api endpoint + + if self.api_endpoint: + return self.function_call_over_api_endpoint(model_name=self.model_name, + context=self.context,params=self.primary_keys, + function=self.function, + api_key=self.api_key,get_logits=self.get_logits) + + # END - route to api endpoint + + prompt = self.fc_prompt_engineer(self.context, params=self.primary_keys, function=self.function) + + # second - tokenize to get the input_ids + + tokenizer_output = self.tokenizer.encode(prompt) + input_token_len = len(tokenizer_output) + input_ids = torch.tensor(tokenizer_output).unsqueeze(0) + + # explicit check and setting to facilitate debugging + if self.use_gpu: + input_ids = input_ids.to('cuda') + else: + input_ids = input_ids.to('cpu') + + # time start + time_start = time.time() + + # This simplified greedy sampling generation loop was derived from and inspired by ideas in the + # HuggingFace transformers library generation class. + # https: //github.com/huggingface/transformers/tree/main/src/transformers/generation + # Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc.team, and NVIDIA Corporation. + # Licensed under the Apache License, Version 2.0 (the "License") + + # default settings + pad_token_id = 0 + + # for most models, eos_token_id = 0, but llama and mistral = 2 + eos_token_id = [self.eos_token_id] + # eos_token_id = [0] + + eos_token_id_tensor = torch.tensor(eos_token_id).to(input_ids.device) + + # keep track of which sequences are already finished + unfinished_sequences = torch.ones(input_ids.shape[0], dtype=torch.long, device=input_ids.device) + + this_peer_finished = False # used by synced_gpus only + # auto-regressive generation + new_tokens_generated = 0 + + attn_mask = torch.ones(input_ids.shape[1]).unsqueeze(0) + + # explicit check and setting to facilitate debugging, if needed + if self.use_gpu: + attn_mask = attn_mask.to('cuda') + else: + attn_mask = attn_mask.to('cpu') + + batch_size = input_ids.shape[0] + seq_len = input_ids.shape[1] + + pkv = None + + while True: + + inp_one_time: torch.LongTensor = input_ids + + if new_tokens_generated > 0: + inp_one_time = input_ids[:, -1:] + + # explicit check and setting to facilitate debugging, if needed + if self.use_gpu: + inp0 = inp_one_time.to('cuda') + inp1 = attn_mask.to('cuda') + else: + inp0 = inp_one_time.to('cpu') + inp1 = attn_mask.to('cpu') + + # inp3 = torch.LongTensor([new_tokens_generated]) + + # need to invoke forward pass on model + # outputs = self.model(inp0,inp1,pkv) + + with torch.no_grad(): + outputs = self.model(input_ids=inp0, attention_mask=inp1, past_key_values=pkv, return_dict=True) + + new_tokens_generated += 1 + + next_token_logits = outputs.logits[:, -1, :] + + # option to capture logits for analysis + # if self.get_logits: self.register_top_logits(next_token_logits) + + if self.temperature and self.sample: + next_token_scores = next_token_logits / self.temperature + else: + next_token_scores = next_token_logits + + # get token from logits + probs = torch.nn.functional.softmax(next_token_scores, dim=-1) + + if not self.sample: + # will pull the 'top logit' only + next_tokens = torch.argmax(probs).unsqueeze(0) + else: + # will apply probabilistic sampling + next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1) + + # option to capture logits and output tokens for analysis + if self.get_logits: + self.register_top_logits(next_token_logits) + + # capture the output tokens + if self.use_gpu: + next_tokens_np = np.array(next_tokens.to('cpu')) + else: + next_tokens_np = np.array(next_tokens) + + self.output_tokens.append(next_tokens_np[0]) + + # finished sentences should have their next token be a padding token + if eos_token_id is not None: + next_tokens = next_tokens * unfinished_sequences + pad_token_id * (1 - unfinished_sequences) + + # update generated ids, model inputs, and length for next step + input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1) + + # testing output in progress starts here + """ + logger.debug(f"update: input_ids - {input_ids}") + # outputs_detached = outputs.to('cpu') + outputs_np = np.array(input_ids[0]) + output_str = self.tokenizer.decode(outputs_np) + logger.debug(f"update: output string - {output_str}") + """ + # end - testing output in progress + + pkv = outputs.past_key_values + + # update attention mask + attn_mask = torch.cat([attn_mask, attn_mask.new_ones((attn_mask.shape[0], 1))], dim=-1) + + # if eos_token was found in one sentence, set sentence to finished + if eos_token_id_tensor is not None: + unfinished_sequences = unfinished_sequences.mul( + next_tokens.tile(eos_token_id_tensor.shape[0], 1).ne(eos_token_id_tensor.unsqueeze(1)).prod( + dim=0) + ) + + # stop when each sentence is finished + if unfinished_sequences.max() == 0: + this_peer_finished = True + + # stop if we exceed the maximum length + if new_tokens_generated >= self.target_requested_output_tokens: + this_peer_finished = True + + if this_peer_finished: + break + + # Generation completed - prepare the output + + if self.use_gpu: + outputs_np = np.array(input_ids[0].to('cpu')) + else: + outputs_np = np.array(input_ids[0]) + + output_only = outputs_np[input_token_len:] + + output_str = self.tokenizer.decode(output_only) + + # post-processing clean-up - stop at endoftext + eot = output_str.find("<|endoftext|>") + if eot > -1: + output_str = output_str[:eot] + + # new post-processing clean-up - stop at + eots = output_str.find("") + if eots > -1: + output_str = output_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = output_str.find(":") + if bot > -1: + output_str = output_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = output_str.find("") + if boss > -1: + output_str = output_str[boss + len(""):] + + # end - post-processing + + total_len = len(outputs_np) + + usage = {"input": input_token_len, + "output": total_len - input_token_len, + "total": total_len, + "metric": "tokens", + "processing_time": time.time() - time_start} + + try: + output_value = ast.literal_eval(output_str) + + output_type = "dict" + + # allow for multiple valid object types - will expand over time + if isinstance(output_value,dict): output_type = "dict" + if isinstance(output_value,list): output_type = "list" + + usage.update({"type": output_type}) + + except: + # could not convert automatically to python object + output_type = "string" + usage.update({"type": output_type}) + output_value = output_str + + # INSERT NEW HERE + + if self.auto_remediate_function_call_output: + + # attempt to remediate + output_type, output_rem = ModelCatalog().remediate_function_call_string(output_str) + + usage.update({"type": output_type, "remediation": True}) + output_value = output_rem + + if output_type == "string": + logger.warning("update: automatic conversion of function call output failed, and attempt to " + "remediate was not successful - %s ", output_str) + else: + logger.info("update: function call output could not be automatically converted, but remediation " + "was successful to type - %s ", output_type) + + # INSERT ENDS HERE + + output_response = {"llm_response": output_value, "usage": usage} + + if get_logits: + output_response.update({"logits": self.logits_record}) + output_response.update({"output_tokens": self.output_tokens}) + self.logits = self.logits_record + + # output inference parameters + self.llm_response = output_value + self.usage = usage + self.final_prompt = prompt + + self.register() + + return output_response + + def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False): + + """ Called by .inference method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + self.prompt=prompt + self.context=context + + # preview call before invoking inference over rest api + self.preview() + + import ast + import requests + + url = self.api_endpoint + "{}".format("/") + output_raw = requests.post(url, data={"model_name": self.model_name, + "question": self.prompt, + "context": self.context, + "api_key": self.api_key, + "max_output": self.max_output, + "temperature": self.temperature}) + + try: + + output = json.loads(output_raw.text) + + # will attempt to unpack logits - but catch any exceptions and skip + if "logits" in output: + try: + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + except: + output["logits"] = [] + + # will attempt to unpack output tokens - but catch any exceptions and skip + if "output_tokens" in output: + try: + # ot_int = [int(x) for x in output["output_tokens"]] + # output["output_tokens"] = ot_int + output_tokens = ast.literal_eval(output["output_tokens"]) + output["output_tokens"] = output_tokens + except: + output["output_tokens"] = [] + + except: + logger.warning("warning: api inference was not successful") + output = {"llm_response": "api-inference-error", "usage": {}} + + # output inference parameters + self.llm_response = output["llm_response"] + self.usage = output["usage"] + self.final_prompt = prompt + + if "logits" in output: + self.logits = output["logits"] + if "output_tokens" in output: + self.output_tokens = output["output_tokens"] + + self.register() + + return output + + def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="", + function=None, endpoint_base=None, api_key=None, get_logits=False): + + """ Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + self.context = context + self.tool_type = tool_type + self.model_name = model_name + + # send to api agent server + + import ast + import requests + + if endpoint_base: + self.api_endpoint = endpoint_base + + if api_key: + # e.g., "demo-test" + self.api_key = api_key + + if not params: + self.model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type] + mc = ModelCatalog().lookup_model_card(self.model_name) + if "primary_keys" in mc: + params = mc["primary_keys"] + self.primary_keys = params + + if function: + self.function = function + + self.prompt = prompt + + # preview before invoking rest api + self.preview() + + url = self.api_endpoint + "{}".format("/agent") + output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key, + "tool_type": self.tool_type, + "function": self.function, + "params": self.primary_keys, "max_output": 50, + "temperature": 0.0, "sample": False, "prompt": self.prompt, + "context": self.context, "get_logits": True}) + + try: + # output = ast.literal_eval(output_raw.text) + output = json.loads(output_raw.text) + if "logits" in output: + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + + if "output_tokens" in output: + ot_int = [int(x) for x in output["output_tokens"]] + output["output_tokens"] = ot_int + + # need to clean up logits + except: + logger.warning("warning: api inference was not successful") + output = {} + + logger.info(f"TEST: executed Agent call over API endpoint - {model_name} - {function} - {output}") + + # output inference parameters + self.llm_response = output["llm_response"] + self.usage = output["usage"] + self.final_prompt = prompt + + if "logits" in output: + self.logits = output["logits"] + if "output_tokens" in output: + self.output_tokens = output["output_tokens"] + + self.register() + + return output + + +class GGUFGenerativeModel(BaseModel): + + """ Implementation of GGUF Model class - instantiate and run inferences and function calls using + GGUF llama.cpp models """ + + def __init__(self, model_name=None, model_card=None, api_key=None, prompt_wrapper=None, instruction_following=False, + context_window=2048, use_gpu_if_available=True, get_logits=False, + sample=True, max_output=100, temperature=0.3, api_endpoint=None, **kwargs): + + super().__init__(**kwargs) + + logger.debug("GGUFGenerativeModel - constructing GGUF model.") + + self.model_class = "GGUFGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.prompt = None + self.final_prompt = None + + # set verbose level in environ level - will be picked up by callback in llama_cpp + os.environ["llama_cpp_verbose"] = GGUFConfigs().get_config("llama_cpp_verbose") + # os.environ["llama_cpp_verbose"] = "ON" + # adding new parameters - use_sampling, temperature, max_output + + self.use_sampling = sample + self.sample = sample + + self.get_logits = get_logits + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.auto_remediate_function_call_output = True + + # default safety check in GGUF Configs that can be adjusted + gguf_configs_max = GGUFConfigs().get_config("max_output_tokens") + + if max_output > gguf_configs_max: + # truncate max output to GGUFConfigs max + # logger.warning(f"update: requested output len - {max_output} > {gguf_configs_max}, which is the " + # f"current GGUF default max.\n--Truncating to {gguf_configs_max} output tokens.\n--Note: " + # f"to change GGUF default max to new integer amount, say 500:\n " + # f" GGUFConfigs().set_config(\"max_output_tokens\", 500)" + # ) + + max_output = gguf_configs_max + + self.max_output = max_output + self.n_seq_max = max_output + + self.target_requested_output_tokens = self.n_seq_max + + self.max_total_len = 2048 + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + self.max_output_len = self.n_seq_max + + self.model_name = model_name + self.prompt_wrapper = prompt_wrapper + self.instruction_following = instruction_following + self.trailing_space = "" + self.separator = "\n" + self.eos_token_id = 0 + + self.add_prompt_engineering = False + self.add_context = "" + self.model_type = "gguf" + self.model_card = model_card + + self.gguf_file = None + self.gguf_repo = None + self.primary_keys = None + self.function = None + self.hf_tokenizer_name = None + self.fc_supported = False + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "tokenizer" in model_card: + self.hf_tokenizer_name = model_card["tokenizer"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + else: + self.trailing_space = "" + + if "eos_token_id" in model_card: + self.eos_token_id = model_card["eos_token_id"] + + if "context_window" in model_card: + self.max_total_len = model_card["context_window"] + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + if "gguf_file" in model_card: + self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf" + + if "gguf_repo" in model_card: + self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf" + + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + + # temperature configuration + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at GGUFConfigs default + self.temperature = GGUFConfigs().get_config("temperature_default") + + # gguf specific attributes + + self._lib = None + self._model = None + self._ctx = None + self._batch = None + self.model_path = None + self.model_params = None + self.context_params = None + + # new option to 'force' use of cuda lib, and over-ride safety checks + if GGUFConfigs().get_config("force_gpu"): + self.use_gpu = True + else: + if sys.platform.lower() not in GGUFConfigs().get_config("cuda_platforms"): + self.use_gpu = False + else: + # min drivers set to the lowest level for CUDA 12.1 on Linux + min_drivers = [525, 60] + if sys.platform.lower() == "win32": + min_drivers = GGUFConfigs().get_config("cuda_windows_driver_min") + + gpu_available = ModelCatalog().gpu_available(driver_min_levels=min_drivers) + + # use_gpu set to TRUE only if: + # (1) cuda_platform (e.g., linux or win32), e.g., not set on Mac OS + # (2) use_gpu set to True in GGUFConfigs + # (3) use_gpu_if_available flag set to True (by default) + # (4) cuda found and drivers current via direct polling of nvidia-smi executable in + # ModelCatalog.gpu_available method + + self.use_gpu = (GGUFConfigs().get_config("use_gpu") + and sys.platform.lower() in GGUFConfigs().get_config("cuda_platforms") + and gpu_available["drivers_current"] and gpu_available["gpu_found"] + and use_gpu_if_available) + + # set default minimum + self.n_batch = 2048 + # self.n_batch = 512 + + self.last_n_tokens_size = 64 + + # by default + self._logits_all = False + + self._n_vocab = None + self._n_ctx = None + self._token_nl = None + self._token_eos = None + self._candidates = None + self.input_ids = None + self.scores = None + self.n_tokens = 0 + self.prev = [] + self.grammar = None + + for key, value in GGUFConfigs().get_sampling_params().items(): + setattr(self, key, value) + + # no api key expected or required + self.api_key = api_key + self.api_endpoint = api_endpoint + + self.error_message = "\nUnable to identify and load GGUF Generative model." + + self.prompt = "" + self.context = "" + self.tool_type = None + + self.model_repo_path = None + + self._sampler = None + self.vocab = None + + self.input_token_count = 0 + self.output_token_count = 0 + + self.post_init() + + def load_model_for_inference(self, model_repo_path, model_card=None, **kwargs): + + """ Loads and instantiates model along with other required objects. """ + + self.model_repo_path = model_repo_path + + if model_card: + self.model_card = model_card + + # validate before loading + self.validate() + + # load shared library + self._lib = self._load_llama_cpp_shared_library() + + self._lib = add_ctypes_declarations(self._lib) + + if not GGUFConfigs().get_config("backend_initialized"): + # is this backend init required? + self._lib.llama_backend_init() + GGUFConfigs().set_config("backend_initialized", True) + + self._lib.llama_log_set(llama_log_callback, ctypes.c_void_p(0)) + + self.model_params = self._lib.llama_model_default_params() + + # update model params parameters + # important to set this correctly for Mac performance + self.model_params.n_gpu_layers = 50 + + # deprecated - change default split_mode from 1 -> 0 + # self.model_params.split_mode = 0 + + self.model_params.main_gpu = 0 + self.model_params.vocab_only = False + self.model_params.use_mmap = True + self.model_params.use_mlock = False + + if self.use_gpu: + # on darwin, keep at 0 - on win32 and linux - set to 50 by default (e.g., shift all model layers to GPU) + if sys.platform.lower() == "win32" or sys.platform.lower().startswith("linux"): + self.model_params.n_gpu_layers = GGUFConfigs().get_config("n_gpu_layers") + + # update context parameters + self.context_params = self._lib.llama_context_default_params() + + # sets minimum of 2048, but will extend if context_window is larger (e.g., 4096/8192+) + self.context_params.n_ctx = max(2048, self.max_total_len) + self.context_params.n_batch = self.n_batch + + n_ubatch = 512 + self.context_params.n_ubatch = min(self.n_batch, n_ubatch) + + # check on QC/ARM if 6 & 12 are ideal + # big improvement on MAC with formula below + # QC/ARM = 6 + import multiprocessing + + self.context_params.n_threads = max(multiprocessing.cpu_count() // 2, 1) + # QC/ARM = 12 + self.context_params.n_threads_batch = multiprocessing.cpu_count() + + self.context_params.rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED + self.context_params.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED + self.context_params.rope_freq_base = 0.0 # (rope_freq_base if rope_freq_base != 0.0 else 0) + self.context_params.rope_freq_scale = 0.0 + + # changed: defaults changed in llama cpp from build b6323 -> b6325 + # self.context_params.yarn_ext_factor = -1.0 + # self.context_params.yarn_attn_factor = 1.0 + # self.context_params.yarn_beta_fast = 32.0 + # self.context_params.yarn_beta_slow = 1.0 + # end changes + + self.context_params.type_k = 1 + self.context_params.type_v = 1 + self.context_params.offload_kqv = True + self.context_params.yarn_orig_ctx = 0 + self.context_params.no_perf = False + + # changes - llama cpp change from b6323 -> b6325 + self.context_params.flash_attn = 0 # False + # self.context_params.flash_attn_type = 0 + # end changes + + self.context_params.embedding = False + self.context_params.swa_full = None + self.context_params.op_offloat = None + + self.context_params.kv_unified = False + + if model_card: + self.model_name = model_card["model_name"].split("/")[-1] + self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf", + self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf" + + self.model_path = os.path.join(model_repo_path, self.gguf_file) + + # loads and instantiates the key objects + self._model = _LlamaModel(self._lib, path_model=self.model_path, params=self.model_params) + + self._ctx = _LlamaContext(self._lib, model=self._model, params=self.context_params) + + self._batch = _LlamaBatch(self._lib, n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx) + + self.vocab = self._lib.llama_model_get_vocab(self._model.model) + + self._n_vocab = self.n_vocab() + self._n_ctx = self.n_ctx() + + self._token_nl = self.token_nl() + self._token_eos = self.token_eos() + + self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab) + + self.input_ids = np.ndarray((self._n_ctx,), dtype=np.intc) + self.scores = np.ndarray((self._n_ctx, self._n_vocab), dtype=np.single) + + self._sampler = self._init_sampler() + + logger.info("GGUFGenerativeModel - loaded model - ready for inference") + + return self + + def _load_llama_cpp_shared_library(self): + + """ Loads llama_cpp shared library - checks if a custom lib path has been configured - otherwise, + it loads the llmware provided dynamic libraries based on the platform/system. """ + + # check first if custom_lib_path - expected to be full path to custom so/dylib file + custom_path = GGUFConfigs().get_config("custom_lib_path") + cdll_args = dict() + + # add option to fall_back if CUDA driver can not be loaded correctly to CPU driver for that OS + fall_back_option = "" + + if custom_path: + + if os.path.exists(custom_path): + _lib_paths = [custom_path] + else: + raise LLMWareException(message="ModuleNotFound error: could not find location of custom lib") + + else: + + _base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf") + + _lib_paths = [] + + system_platform = sys.platform.lower() + + # Determine the file extension based on the platform + if system_platform.startswith("linux"): + + # three linux versions supported - linux_x86 and linux_cuda + machine = os.uname().machine.lower() + + if machine == "aarch64" and self.use_gpu: + _lib_paths.append(os.path.join(_base_path, + GGUFConfigs().get_config("linux_aarch64_cuda_lib"), + GGUFConfigs().get_config("linux_cuda"))) + + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"), + GGUFConfigs().get_config("linux_cuda"))) + + # will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("linux_x86")) + + else: + # by default load the cpu x86 lib + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("linux_x86"))) + + elif system_platform == "darwin": + + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"), + GGUFConfigs().get_config("mac_metal"))) + + elif sys.platform == "win32": + + import platform + if platform.machine().lower() == "arm64": + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"), + GGUFConfigs().get_config("windows_arm64"))) + + # windows cuda + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"), + GGUFConfigs().get_config("windows_cuda"))) + + # new - will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("windows")) + + else: + # main case - windows x86 + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("windows"))) + + else: + raise LLMWareException(message=f"No matching llama.cpp binary for platform - {system_platform}") + + # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32" and sys.version_info >= (3, 8): + os.add_dll_directory(str(_base_path)) + + # need to review + if "CUDA_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) + + cdll_args["winmode"] = ctypes.RTLD_GLOBAL + + # Try to load the shared library, handling potential errors + for _lib_path in _lib_paths: + + logger.debug(f"Loading llama cpp backend - {_lib_path}") + + if not os.path.exists(_lib_path): + if fall_back_option: + _lib_path = fall_back_option + + if os.path.exists(_lib_path): + + try: + return ctypes.cdll.LoadLibrary(str(_lib_path)) + + except Exception as e: + + # if fail, and CUDA selected, then try to fall back to matching CPU version + if fall_back_option: + try: + + logger.warning("Not successful loading preferred lib so reverting to fallback lib.") + + return ctypes.cdll.LoadLibrary(str(_lib_path)) + except: + + # if fall-back fails + raise GGUFLibNotLoadedException("llama_cpp_backend", + sys.platform.lower(), + self.use_gpu, + _lib_path, + custom_path) + else: + raise GGUFLibNotLoadedException("llama_cpp_backend" ,sys.platform.lower(), + self.use_gpu, _lib_path, custom_path) + + # if not loaded + raise LLMWareException(message=f"GGUFGenerativeModel - attempting to load llama cpp backend lib - " + f"Llama cpp backend not found.") + + def _init_sampler(self): + + # create sampler + # default params are struct + params = llama_sampler_chain_params() + self._sampler = self._lib.llama_sampler_chain_init(params) + + temp = 0.0 + + if temp < 0.0: + # sampler.add_softmax() + self._lib.llama_sampler_chain_add(self._sampler, self._lib.llama_sampler_init_softmax()) + # sampler.add_dist(self._seed) + + elif temp == 0.0: + # sampler.add_greedy() + greedy_sampler = self._lib.llama_sampler_init_greedy() + + self._lib.llama_sampler_chain_add(self._sampler, greedy_sampler) + + return self._sampler + + def sample_gguf(self, idx=None): + + """ Adapted to sample_gguf to avoid potential name space conflicts. """ + + # assert self.n_tokens > 0 + + tmp_sampler = False + + if self._sampler is None: + tmp_sampler = True + self._sampler = self._init_sampler() + + ridx = idx - self.n_tokens if idx is not None else -1 + + assert self.ctx is not None + + token = self._lib.llama_sampler_sample(self._sampler, self._ctx.ctx, ridx) + + # token = int(self.logits_record[-1][0][0]) + + if tmp_sampler: + self._sampler = None + return token + + def _inference(self, prompt): + + """ Tokenizes the prompt and executes generation loop. """ + + t0 = time.time() + + completion_tokens = [] if len(prompt) > 0 else [self.token_bos()] + + prompt_tokens = ( + ( + self.tokenize(prompt.encode("utf-8"), special=True) + if prompt != "" + else [self.token_bos()] + ) + if isinstance(prompt, str) + else prompt + ) + + # confirm that input is smaller than context_window + input_len = len(prompt_tokens) + context_window = self.n_ctx() + + if input_len > context_window: + logger.info("GGUFGenerativeModel - input is too long for model context window - truncating") + min_output_len = 10 + prompt_tokens = prompt_tokens[0:context_window - min_output_len] + input_len = len(prompt_tokens) + + text = b"" + + # first token capture starts here + get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed") + + token_counter = 0 + t_gen_start = time.time() + first_token_processing_time = -1.0 + + for token in self.generate(prompt_tokens): + + # first token capture + if get_first_token_speed: + if token_counter == 0: + first_token_processing_time = time.time() - t_gen_start + token_counter += 1 + # first token capture ends here + + if self.get_logits: + self.register_top_logits() + self.output_tokens.append(token) + + if token == self._token_eos: + text = self.detokenize(completion_tokens) + break + + completion_tokens.append(token) + + # stop at max output len + if len(completion_tokens) >= self.max_output_len: + text = self.detokenize(completion_tokens) + break + + # stop if combined input + output at context window size + if (input_len + len(completion_tokens)) >= context_window: + text = self.detokenize(completion_tokens) + break + + text_str = text.decode("utf-8", errors="ignore") + + # post-processing clean-up - stop at endoftext + eot = text_str.find("<|endoftext|>") + if eot > -1: + text_str = text_str[:eot] + + # new post-processing clean-up - stop at + eots = text_str.find("") + if eots > -1: + text_str = text_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = text_str.find(":") + if bot > -1: + text_str = text_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = text_str.find("") + if boss > -1: + text_str = text_str[boss + len(""):] + + # end - post-processing + + if get_first_token_speed: + + output = {"llm_response": text_str, + "usage": {"input": len(prompt_tokens), "output": len(completion_tokens), + "total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens", + "processing_time": time.time() - t0, + "first_token_processing_time": first_token_processing_time}} + else: + output = {"llm_response": text_str, + "usage": {"input": len(prompt_tokens), "output": len(completion_tokens), + "total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens", + "processing_time": time.time() - t0}} + + if self.get_logits: + output.update({"logits": self.logits_record}) + output.update({"output_tokens": self.output_tokens}) + + return output + + def generate(self, tokens, reset=True): + + """ Generator that samples the model and yields tokens until stopped. """ + + logger.debug("GGUFGenerativeModel - starting generation loop") + + # Reset the model state + if reset: + self.reset() + + sample_idx = self.n_tokens + len(tokens) - 1 + tokens = list(tokens) + + tokens_created = 0 + input_start_len = len(tokens) + + memory = self._ctx.memory + + # Eval and sample + while True: + + self._lib.llama_memory_seq_rm(memory, -1, self.n_tokens, -1) + + for i in range(0, len(tokens), self.n_batch): + batch = tokens[i: min(len(tokens), i + self.n_batch)] + n_past = self.n_tokens + n_tokens = len(batch) + + self._batch.set_batch(batch=batch, n_past=n_past, logits_all=self._logits_all) + + return_code = self._lib.llama_decode(self._ctx.ctx, self._batch.batch) + + # TODO: add better error handling if return_code 1 - usually overflow of ctx + if return_code != 0: + raise RuntimeError(f"GGUFGenerativeModel - generate - llama_decode call returned {return_code} - in most cases, this " + f"is due to exceeding the maximum context window.") + + self.input_ids[n_past: n_past + n_tokens] = batch + rows = n_tokens + cols = self._n_vocab + offset = (0 if self._logits_all else n_tokens - 1) + + if self._logits_all: + rows = n_tokens + cols = self._n_vocab + logits = np.ctypeslib.as_array( + self._ctx.get_logits(), shape=(rows * cols,)) + self.scores[n_past: n_past + n_tokens, :].reshape(-1)[::] = logits + + self.n_tokens += n_tokens + + # TODO: inserting test for logits + # self.register_top_logits() + + while sample_idx < self.n_tokens: + + logits = self._scores[-1, :] + + self.prev = list(self.eval_tokens) + + # note: call to .sample_gguf method + token = self.sample_gguf(idx=sample_idx) # (logits_array=logits) + + self.accept(id=id, apply_grammar=None) + + tokens_created += 1 + + sample_idx += 1 + + tokens_or_none = yield token + tokens.clear() + tokens.append(token) + if tokens_or_none is not None: + tokens.extend(tokens_or_none) + + if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]: + self.n_tokens = sample_idx + + self._lib.llama_memory_seq_rm(self._lib.llama_get_memory(self._ctx.ctx), -1, self.n_tokens, -1) + # self._lib.llama_kv_cache_seq_rm(self._ctx.ctx, -1, self.n_tokens, -1) + + break + + if tokens_created > self.max_output_len: + logger.info("GGUFGenerativeModel - stopping generation loop - reached limit of " + "max output len") + break + + def tokenize(self, text, add_bos=True, special=False): + + """ Tokenizes text. """ + + n_ctx = self.n_ctx_train() + tokens = (ctypes.c_int32 * n_ctx)() + # change from self._model.model + n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_ctx, add_bos, special) + + if n_tokens < 0: + n_tokens = abs(n_tokens) + tokens = (ctypes.c_int32 * n_tokens)() + + n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_tokens, add_bos, special) + + if n_tokens < 0: + raise RuntimeError(f"GGUFGenerativeModel - tokenization error - {text} - " + f"n_tokens={n_tokens}") + + return list(tokens[:n_tokens]) + + def detokenize(self, tokens, special: bool = False) -> bytes: + output = b"" + size = 32 + buffer = (ctypes.c_char * size)() + for token in tokens: + n = self._lib.llama_token_to_piece( + # replace: self.model + self.vocab, llama_token(token), buffer, size, 0, special + ) + assert n <= size + output += bytes(buffer[:n]) + + # NOTE: Llama1 models automatically added a space at the start of the prompt + # this line removes a leading space if the first token is a beginning of sentence token + + return ( + output[1:] + if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b" " + else output + ) + + def accept(self, id, apply_grammar): + + """ Formal step post sampling that 'accepts' and adds the token id to the running generation. """ + + if apply_grammar and self.grammar is not None: + self._lib.llama_grammar_accept_token(self._ctx.ctx, self.grammar.grammar, id) + + self.prev.append(id) + + def register_top_logits(self): + + """ Gets the top logits and keeps a running log for output analysis. """ + + # TODO: there is issue with first logit computation - not corresponding to first token + logit_pointer = self._lib.llama_get_logits(self._ctx.ctx) + + logit_size = self.n_vocab() + logit_array = np.zeros(logit_size) + for x in range(0, logit_size): + logit_array[x] = logit_pointer[x] + + sm = np.exp(logit_array) / sum(np.exp(logit_array)) + + sm_sorted = np.sort(sm) + sm_args_sorted = np.argsort(sm) + + top_logits = [] + + for x in range(0, self.top_logit_count): + # experiment - try rounding the float number + pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3)) + top_logits.append(pair) + + self.logits_record.append(top_logits) + + return top_logits + + def set_api_key(self, api_key, env_var="USER_MANAGED_GGUF_API_KEY"): + + """ Sets API key - generally not used in GGUF models. """ + + # set api_key + os.environ[env_var] = api_key + logger.info("GGUFGenerativeModel - added and stored GGUF api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_GGUF_API_KEY"): + + """ Gets API key - generally not used in GGUF models. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.error("GGUFGenerativeModel - _get_api_key could not successfully retrieve value from: %s ", env_var) + + return self.api_key + + def token_counter(self, text_sample): + + if not text_sample: + tokens = 0 + else: + tokens = len(self.tokenize(text_sample.encode("utf-8"))) + + return tokens + + @property + def ctx(self): + return self._ctx.ctx + + @property + def model(self): + return self._model.model + + @property + def _input_ids(self): + return self.input_ids[: self.n_tokens] + + @property + def _scores(self): + return self.scores[: self.n_tokens, :] + + @property + def eval_tokens(self): + return deque(self.input_ids[: self.n_tokens].tolist(), maxlen=self._n_ctx) + + @property + def eval_logits(self): + return deque( + self.scores[: self.n_tokens, :].tolist(), + maxlen=self._n_ctx if self._logits_all else 1, + ) + + def reset(self): + self.n_tokens = 0 + + def n_ctx(self): + return self._lib.llama_n_ctx(self._ctx.ctx) + + def n_ctx_train(self): + return self._lib.llama_n_ctx_train(self._model.model) + + def n_vocab(self): + n_vocab = self._lib.llama_n_vocab(self._lib.llama_model_get_vocab(self._model.model)) + return n_vocab + + def token_eos(self): + eos = self._lib.llama_token_eos(self.vocab) + return eos + + def token_bos(self): + bos = self._lib.llama_token_bos(self.vocab) + return bos + + def token_nl(self): + token_nl = self._lib.llama_token_nl(self._lib.llama_model_get_vocab(self._model.model)) + return token_nl + + def unload_model(self): + + """ Unloads a model to release memory """ + + # note: removing pointer seems to safely remove from Python reference tracking + # --will evaluate under multiple scenarios if free explicitly needs to be called in llama.cpp engine + + self._batch = None + self._ctx = None + self._model = None + + return 0 + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, inference_dict=None, + get_logits=False): + + """ Main method for inference generation. """ + + self.prompt = prompt + + # first prepare the prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # update default handling for no add_prompt_engineering + + if not self.add_prompt_engineering: + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # end - update + + # show warning if function calling model + if self.fc_supported: + logger.info("GGUFGenerativeModel - this is a function calling model - using .inference may lead to unexpected " + "results. Recommended to use the .function_call method to ensure correct prompt " + "template packaging.") + + # start with clean logits_record and output_tokens for each function call + self.logits_record = [] + self.output_tokens = [] + + if get_logits: + self.get_logits = get_logits + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # preview before initiating inference over api + self.preview() + + # START - route to api endpoint + if self.api_endpoint: + sd = self.to_state_dict() + return self.inference_over_api_endpoint(self.prompt, context=self.add_context, + inference_dict=inference_dict) + + # END - route to api endpoint + + text_prompt = self.prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # text_prompt = prompt_final + "\n" + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + text_prompt = prompt_final + self.trailing_space + + output_response = self._inference(text_prompt) + + # update linked to BaseModel + self.prompt = prompt + self.final_prompt = text_prompt + self.usage = output_response["usage"] + self.llm_response = output_response["llm_response"] + + if "logits" in output_response: + self.logits = output_response["logits"] + + self.register() + # end - update + + return output_response + + def function_call(self, context, function=None, params=None, get_logits=True, + temperature=-99.0, max_output=None): + + """ This is the key inference method for SLIM models - takes a context passage and a key list + which is packaged in the prompt as the keys for python dictionary output""" + + if not self.fc_supported: + logger.warning("GGUFGenerativeModel - loaded model does not support function calls. " + "Please either use the standard .inference method with this model, or use a GGUF " + "model that has 'function_calls' key set to True in its model card.") + return [] + + self.context = context + + # start with clean logits_record and output_tokens for each function call + self.logits_record = [] + self.output_tokens = [] + + if get_logits: + self.get_logits = get_logits + + if params: + self.primary_keys = params + + if not self.primary_keys: + logger.warning("GGUFGenerativeModel - function call - no keys provided - " + "function call may yield unpredictable results") + + if not params: + params = self.primary_keys + + if not function: + # pull from model card + if self.function: + if isinstance(self.function, list): + if len(self.function) > 0: + function = self.function[0] + else: + function = self.function + + if not function: + function = "classify" + + self.primary_keys = params + self.function = function + + # preview before initiating api call + self.preview() + + # START - route to api endpoint + + if self.api_endpoint: + + return self.function_call_over_api_endpoint(model_name=self.model_name, + context=self.context,params=self.primary_keys, + function=self.function, + api_key=self.api_key,get_logits=self.get_logits) + + # END - route to api endpoint + + # prepare SLIM prompt + class_str = "" + for key in params: + class_str += str(key) + ", " + if class_str.endswith(", "): + class_str = class_str[:-2] + + f = str(self.function) + + full_prompt = ": " + self.context + "\n" + "<{}> {} ".format(f, class_str, f) + "\n:" + full_prompt = full_prompt + self.trailing_space + + text_prompt = full_prompt + + if temperature != -99: + self.temperature = temperature + + if max_output: + self.max_output_len = max_output + + # call inference here + output_response = self._inference(text_prompt) + + output_str = output_response["llm_response"] + + try: + import ast + output_dict = ast.literal_eval(output_str) + + output_type = "dict" + if isinstance(output_dict, dict): output_type = "dict" + if isinstance(output_dict, list): output_type = "list" + + output_response["usage"].update({"type": output_type}) + output_response.update({"llm_response": output_dict}) + + except: + + output_type = "string" + output_response["usage"].update({"type": output_type}) + + if self.auto_remediate_function_call_output: + + # attempt to automatically remediate + output_type, output_rem = ModelCatalog().remediate_function_call_string(output_str) + + if output_type != "string": + output_response["usage"].update({"type": output_type, "remediation": True}) + output_response.update({"llm_response": output_rem}) + + if output_type == "string": + logger.warning("GGUFGenerativeModel - function call - automatic conversion of function call output failed, and attempt to " + "remediate was not successful - %s ", output_str) + else: + logger.info("GGUFGenerativeModel - function call output could not be automatically converted, but remediation " + "was successful to type - %s ", output_type) + + # update linked to BaseModel + self.prompt = "" + self.final_prompt = full_prompt + self.usage = output_response["usage"] + self.llm_response = output_response["llm_response"] + + if "logits" in output_response: + self.logits = output_response["logits"] + + self.register() + # end - update + + return output_response + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, api_key=None, inference_dict=None, + get_logits=False, disable_eos=False, skip_pe_override=False): + + """ Main method for text streaming generation. Returns a generator function that yields one + token at a time for real-time streaming to console or UI. """ + + # first prepare the prompt + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # update default handling for no add_prompt_engineering + + if not self.add_prompt_engineering: + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # show warning if function calling model + if self.fc_supported: + logger.info("GGUFGenerativeModel - this is a function calling model - using .inference may lead to unexpected " + "results. Recommended to use the .function_call method to ensure correct prompt " + "template packaging.") + + # start with clean logits_record and output_tokens for each function call + self.logits_record = [] + self.output_tokens = [] + + if get_logits: + self.get_logits = get_logits + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # preview before generation + self.preview() + + if self.add_prompt_engineering and not skip_pe_override: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + prompt = prompt_final + self.trailing_space + + if self.api_endpoint: + """ Not implemented """ + # continue with local execution ... + + # starts _inference here + completion_tokens = [] if len(prompt) > 0 else [self.token_bos()] + + logger.info(f"GGUFGenerative - stream - model name - {self.model_name}") + + prompt_tokens = ( + ( + self.tokenize(prompt.encode("utf-8"), special=True) + if prompt != "" + else [self.token_bos()] + ) + if isinstance(prompt, str) + else prompt + ) + + # confirm that input is smaller than context_window + input_len = len(prompt_tokens) + context_window = self.n_ctx() + + logger.info(f"GGUFGenerativeModel stream - input token len - {input_len}") + + if input_len > context_window: + logger.warning("GGUFGenerativeModel - input is too long for model context window - truncating") + min_output_len = 10 + prompt_tokens = prompt_tokens[0:context_window - min_output_len] + input_len = len(prompt_tokens) + + text = b"" + + # disable_eos = True + token_list = [] + + for token in self.generate(prompt_tokens): + + completion_tokens.append(token) + + if not disable_eos: + if token == self._token_eos: + break + + if len(completion_tokens) > self.max_output_len: + break + + # stop if combined input + output at context window size + if (input_len + len(completion_tokens)) >= context_window: + break + + new_token = self.detokenize([token]).decode('utf-8', errors='ignore') + + # a little cleanup of 'think' tokens + if new_token == "": + new_token = "<|think|>" + logger.info(f"GGUFGenerativeModel - stream - changing token to markdown safe - {new_token}") + + if new_token == "": + new_token = "<|endthink|>" + + yield new_token + + text_str = text.decode("utf-8", errors="ignore") + + # turned off + self.register() + + return text_str + + def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="", + function=None, endpoint_base=None, api_key=None, get_logits=False): + + """ Called by .function_call method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + # send to api agent server + + self.context = context + self.tool_type = tool_type + + import ast + import requests + + if endpoint_base: + self.api_endpoint = endpoint_base + + if api_key: + # e.g., "demo-test" + self.api_key = api_key + + if not params: + self.model_name = _ModelRegistry().get_llm_fx_mapping()[tool_type] + mc = ModelCatalog().lookup_model_card(self.model_name) + if "primary_keys" in mc: + params = mc["primary_keys"] + + if function: + self.function = function + + self.prompt = prompt + + self.primary_keys = params + + # preview before invoking api + self.preview() + + url = self.api_endpoint + "{}".format("/agent") + output_raw = requests.post(url, data={"model_name": self.model_name, "api_key": self.api_key, + "tool_type": self.tool_type, + "function": self.function, "params": self.primary_keys, + "max_output": 50, + "temperature": 0.0, "sample": False, "prompt": self.prompt, + "context": self.context, "get_logits": True}) + + try: + + output = json.loads(output_raw.text) + + # will attempt to unpack logits - but catch any exceptions and skip + if "logits" in output: + try: + import ast + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + except: + output["logits"] = [] + + # will attempt to unpack output tokens - but catch any exceptions and skip + if "output_tokens" in output: + try: + ot_int = [int(x) for x in output["output_tokens"]] + output["output_tokens"] = ot_int + # output_tokens = ast.literal_eval(output["output_tokens"]) + # output["output_tokens"] = output_tokens + except: + output["output_tokens"] = [] + + # output = ast.literal_eval(output_raw.text) + + except: + logger.warning("GGUFGenerativeModel - function_call_over_api_endpoint - api inference was not successful") + output = {"llm_response": "api-inference-error", "usage": {}} + + # update linked to BaseModel + self.prompt = prompt + self.final_prompt = prompt + self.usage = output["usage"] + self.llm_response = output["llm_response"] + + if "logits" in output: + self.logits = output["logits"] + + self.register() + # end - update + + return output + + def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False): + + """ Called by .inference method when there is an api_endpoint passed in the model constructor. Rather + than execute the inference locally, it will be sent over API to inference server. """ + + self.prompt = prompt + self.context = context + + # preview before invoking inference over rest api + self.preview() + + import ast + import requests + + url = self.api_endpoint + "{}".format("/") + output_raw = requests.post(url, data={"model_name": self.model_name, + "question": self.prompt, + "context": self.context, + "api_key": self.api_key, + "max_output": self.max_output_len, + "temperature": self.temperature}) + + try: + output = json.loads(output_raw.text) + + # will attempt to unpack logits - but catch any exceptions and skip + if "logits" in output: + try: + import ast + logits = ast.literal_eval(output["logits"]) + output["logits"] = logits + except: + output["logits"] = [] + + # will attempt to unpack output tokens - but catch any exceptions and skip + if "output_tokens" in output: + try: + import ast + # ot_int = [int(x) for x in output["output_tokens"]] + # output["output_tokens"] = ot_int + output_tokens = ast.literal_eval(output["output_tokens"]) + output["output_tokens"] = output_tokens + except: + output["output_tokens"] = [] + + except: + logger.warning("warning: api inference was not successful") + output = {"llm_response": "api-inference-error", "usage": {}} + + # update linked to BaseModel + self.prompt = prompt + self.final_prompt = prompt + self.usage = output["usage"] + self.llm_response = output["llm_response"] + + if "logits" in output: + self.logits = output["logits"] + + self.register() + # end - update + + return output + + +class WhisperCPPModel(BaseModel): + + """ WhisperCPPModel is an implementation of the Whisper voice transcription model running on GGML, rather + than Pytorch. """ + + def __init__(self, model_name=None, model_card=None, use_gpu_if_available=True, **kwargs): + + super().__init__(**kwargs) + + self.model_class = "WhisperCPPModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.prompt = None + self.final_prompt = None + + # set verbose level in environ level - will be picked up by callback in whisper_cpp + os.environ["whisper_cpp_verbose"] = GGUFConfigs().get_config("whisper_cpp_verbose") + self.WHISPER_SR = GGUFConfigs().get_config("whisper_sr") + self.strategy = GGUFConfigs().get_config("whisper_strategy") + self.n_threads = GGUFConfigs().get_config("whisper_threads") + self.language = GGUFConfigs().get_config("whisper_language") + self.format = GGUFConfigs().get_config("whisper_output_format") + self.tiny_diarize = GGUFConfigs().get_config("whisper_tiny_diarize") + self.beam_size = GGUFConfigs().get_config("whisper_beam_size") + self.greedy_best_of = GGUFConfigs().get_config("whisper_greedy_best_of") + self.temperature_inc = GGUFConfigs().get_config("whisper_temperature_inc") + + self.remove_segment_markers = GGUFConfigs().get_config("whisper_remove_segment_markers") + self.model_card = model_card + self.model_name = model_name + self._lib = None + self.model_path = None + self.context = None + self.params = None + self.temperature = 0.0 + self.duration = 0 + self.translate = False + + if sys.platform.lower() == "darwin": + self.whisper_use_legacy_mac = GGUFConfigs().get_config("whisper_use_legacy_mac") + else: + self.whisper_use_legacy_mac = False + + # new option to 'force' use of cuda lib, and over-ride safety checks + if GGUFConfigs().get_config("force_gpu"): + self.use_gpu = True + else: + if not sys.platform.lower().startswith("linux"): + self.use_gpu = False + else: + # min drivers set to the lowest level for CUDA 12.1 on Linux + min_drivers = [525,60] + + gpu_available = ModelCatalog().gpu_available(driver_min_levels=min_drivers) + + # use_gpu set to TRUE only if: + # (1) cuda_platform (e.g., linux or win32), e.g., not set on Mac OS + # (2) use_gpu set to True in GGUFConfigs + # (3) use_gpu_if_available flag set to True (by default) + # (4) cuda found and drivers current via direct polling of nvidia-smi executable in + # ModelCatalog.gpu_available method + + self.use_gpu = (GGUFConfigs().get_config("use_gpu") + and sys.platform.lower() in GGUFConfigs().get_config("cuda_platforms") + and gpu_available["drivers_current"] and gpu_available["gpu_found"] + and use_gpu_if_available) + + self.model_repo_path = None + + self.post_init() + + def load_model_for_inference(self, model_repo_path, model_card = None, **kwargs): + + """ Loads and instantiates model along with other required objects. """ + + self.model_repo_path = model_repo_path + if model_card: + self.model_card = model_card + + # validate before loading + self.validate() + + # load shared library + self._lib = self._load_shared_library() + self._lib = self.add_ctypes_configs() + + self._lib.whisper_log_set(whisper_log_callback, ctypes.c_void_p(0)) + + if model_card: + self.model_name = model_card["model_name"].split("/")[-1] + self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf", + self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf" + + self.model_path = os.path.join(model_repo_path, self.gguf_file) + self.context = self._lib.whisper_init_from_file(self.model_path.encode('utf-8')) + + self.params = self._lib.whisper_full_default_params(self.strategy) + + self.params.n_threads = self.n_threads + # self.params.print_special = True + # self.params.print_progress = False + + # set to True by default - will display in 'real-time' the transcription + # self.params.print_realtime = GGUFConfigs().get_config("whisper_cpp_realtime_display") + # self.params.print_timestamps = True + # self.params.tdrz_enable = self.tiny_diarize + # self.params.progress_callback = whisper_progress_callback(self.callback) + # self.params.temperature_inc = self.temperature_inc + # self.params.token_timestamps = True + # self.params.greedy.best_of = self.greedy_best_of + # self.params.beam_search.beam_size = self.beam_size + + return self + + def _load_shared_library(self): + + """ Loads the libwhisper.cpp backend GGML engine that runs the model. """ + + # check first if custom_lib_path - expected to be full path to custom so/dylib file + custom_path = GGUFConfigs().get_config("whisper_cpp_lib_path") + fall_back_option = "" + cdll_args = dict() + + if custom_path: + + if os.path.exists(custom_path): + _lib_paths = [custom_path] + else: + raise LLMWareException(message=f"WhisperCPPModel - attempted to load whisper cpp backend lib - " + f"could not find path to custom lib - {custom_path}") + else: + + # general case - will look for llama.cpp dynamic library included with llmware + + _base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf") + + _lib_paths = [] + + system_platform = sys.platform.lower() + + # Determine the file extension based on the platform + if system_platform.startswith("linux"): + + machine = os.uname().machine.lower() + + if machine == "aarch64" and self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_aarch64_cuda_lib"), + GGUFConfigs().get_config("whisper_dgx"))) + + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"), + GGUFConfigs().get_config("whisper_linux_cuda"))) + + # new - will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("whisper_linux_x86")) + + else: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("whisper_linux_x86"))) + + elif system_platform == "darwin": + + if not self.whisper_use_legacy_mac: + mac_lib = GGUFConfigs().get_config("whisper_mac_metal") + else: + mac_lib = GGUFConfigs().get_config("whisper_mac_metal_legacy") + + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"),mac_lib)) + + elif sys.platform == "win32": + + import platform + + if platform.machine().lower() == "arm64": + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"), + GGUFConfigs().get_config("whisper_windows_arm64"))) + + # windows cuda + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"), + GGUFConfigs().get_config("whisper_windows"))) + + # new - will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("whisper_windows")) + + else: + # main case - windows x86 + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("whisper_windows"))) + + # Add the library directory to the DLL search path on Windows (if needed) + # if sys.platform == "win32" and sys.version_info >= (3, 8): os.add_dll_directory(str(_base_path)) + + # Try to load the shared library, handling potential errors + for _lib_path in _lib_paths: + + if not os.path.exists(_lib_path): + if fall_back_option: + _lib_path = fall_back_option + + if os.path.exists(_lib_path): + + try: + return ctypes.CDLL(str(_lib_path), **cdll_args) + + except Exception as e: + + # NEW INSERT - if fail, and CUDA selected, then try to fall back to matching CPU version + if fall_back_option: + try: + logger.warning("update: Not successful loading primary lib, so reverting to secondary " + "driver (which may be slower).") + + return ctypes.CDLL(str(fall_back_option), **cdll_args) + except: + + # if fall-back fails + raise GGUFLibNotLoadedException("whisper_cpp_backend", + sys.platform.lower(), + self.use_gpu, + _lib_path, + custom_path) + else: + raise GGUFLibNotLoadedException("whisper_cpp_backend",sys.platform.lower(), + self.use_gpu, _lib_path, custom_path) + else: + logger.warning(f"update: looking for WhisperCPP lib - path does not exist - {str(_lib_path)}") + + # Try to load the shared library, handling potential errors + # *** something has gone wrong - could not find the lib files + + raise FileNotFoundError(f"Exception: WhisperCPP Shared library not found at paths - {str(_lib_paths)}") + + # new method starts here + + def inference(self, prompt, inference_dict=None): + + """ Inference on Whisper model takes a single input 'prompt' which is a string corresponding to a + full file path pointing to the voice file to be transcribed, e.g., + + `/home/ubuntu/voice_samples/sample.wav + + """ + + self.prompt=prompt + + if inference_dict: + if "translate" in inference_dict: + self.translate=inference_dict["translate"] + + if "remove_segment_markers" in inference_dict: + self.remove_segment_markers = inference_dict["remove_segment_markers"] + + # preview before starting inference + + # self.preview() + + # note: updated dependencies for improved efficiency + # previously, used librosa library + # replaced librosa with two librosa sub-dependencies that do most of the work + # e.g., soundfile, and soxr which results in smaller footprint for deployment + + file = prompt + + if not file.endswith(".wav"): + + logger.info("update: WhisperCPPModel - inference - input file needs to be converted to .wav - " + "will try to do right now.") + + new_file_path = Utilities().convert_media_file_to_wav(self.prompt, + save_path=LLMWareConfig().get_tmp_path(), + file_out="converted_file_tmp.wav") + + if not new_file_path: + logger.warning("update: WhisperCPPModel - inference - conversion was not successful. " + "The most likely causes of this error - \n" + "1. File type is not supported - the following are the supported file types - " + "mp3, m4a, mp4, wma, aac, ogg, flv. \n" + "2. lib ffmpeg is not installed on your system. This is the core audio processing " + "library that handles the file conversion.\n" + "--to install on Mac: brew install ffmpeg \n" + "--to install on Linux: sudo apt install ffmpeg \n" + "--to install on Windows: see ffmpeg.org/download.html for download/install \n") + + null_output = {"llm_response": "", "segments": []} + return null_output + + else: + logger.info(f"update: WhisperCPPModel - inference - file conversion to .wav successful - " + f"new file at tmp path - {new_file_path}") + + file = new_file_path + + # loading new dependencies starts here + + try: + import soundfile as sf + import soxr + except: + raise LLMWareException("WhisperCPPModel class requires dependencies of soundfile and soxr," + "e.g., `pip install soundfile` and `pip install soxr`") + + sfo = sf.SoundFile(file) + + with sfo as sf_desc: + sr = sf_desc.samplerate + frame_duration = -1 + + data = sf_desc.read(frames=frame_duration, dtype=np.float32, always_2d=False).T + + if self.WHISPER_SR != sr: + + # y = resample(data, orig_sr=sr_native, target_sr=sr, res_type="soxr_hq") + + ratio = float(sr) / self.WHISPER_SR + axis = -1 + n_samples = int(np.ceil(data.shape[axis] * ratio)) + + yhat = np.apply_along_axis(soxr.resample, axis=axis, arr=data, + in_rate=sr, out_rate=self.WHISPER_SR, quality="soxr_hq") + + data = np.asarray(yhat, dtype=np.float32) + + # new dependencies end here + # replacing previous: data, sr = librosa.load(file, sr=self.WHISPER_SR) + + try: + self.duration = float(data.shape[-1]) / self.WHISPER_SR + # self.duration = librosa.get_duration(y=data, sr=self.WHISPER_SR) + except: + self.duration = float(0.0) + + data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)) + self.params.language = self.language.encode('utf-8') + if prompt: + self.params.initial_prompt = prompt.encode('utf-8') + + # self.params.temperature = self.temperature + # self.params.translate = self.translate + + result = self._generate(data) + + # output format options + + output = result["text"] + + if self.format == "srt": + output = '\n'.join([f'{i + 1}\n{self._format_time(s["start"])} --> ' + f'{self._format_time(s["end"])}\n{s["text"]}\n' + for i, s in enumerate(result["segments"])]) + + if self.format == "vtt": + output = '\n'.join([f'{i + 1}\n{self._format_time(s["start"])} --> ' + f'{self._format_time(s["end"])} align:middle\n{s["text"]}\n' + for i, s in enumerate(result["segments"])]) + + usage_dict = {"duration-seconds": self.duration, "segments": len(result["segments"]), + "language": self.language} + + response = {"llm_response": output, "usage": usage_dict, "segments": result["segments"]} + + # update linked to BaseModel + self.prompt = "" + self.final_prompt = "" + self.usage = response["usage"] + self.llm_response = response["llm_response"] + + self.register() + # end - update + + return response + + def _generate(self, data): + + """ Executes lib_whisper generation on data from audio file. """ + + w = self._lib.whisper_full(ctypes.c_void_p(self.context), + self.params, + data.ctypes.data_as(ctypes.POINTER(ctypes.c_float)), + len(data)) + + if w != 0: + raise LLMWareException(message=f"Exception: WhisperCPPModel - inference: {w}") + + segments = [] + all_text = "" + text_chunks = [] + n_segments = self._lib.whisper_full_n_segments(ctypes.c_void_p(self.context)) + + for i in range(n_segments): + + t0 = self._lib.whisper_full_get_segment_t0(ctypes.c_void_p(self.context), i)/100.0 + t1 = self._lib.whisper_full_get_segment_t1(ctypes.c_void_p(self.context), i)/100.0 + txt = self._lib.whisper_full_get_segment_text(ctypes.c_void_p(self.context), i).decode('utf-8-sig', + errors='ignore') + + if self.tiny_diarize: + + # look for [_SOLM_] token to break segment - and will keep aggregating until found + if "[_SOLM_]" in txt: + txt += "\n\n" + + if self.remove_segment_markers: + + # removes leading [_BEG_] & trailing [_TT_XYZ] special tokens + + txt_split = txt.split("[_TT_")[0] + txt_split = txt_split.strip() + if txt_split.startswith("[_BEG_]"): + txt_split= txt_split[len("[_BEG_]"):] + txt = " " + txt_split + " " + + all_text += txt + text_chunks.append(txt) + + n_tokens = self._lib.whisper_full_n_tokens(ctypes.c_void_p(self.context), i) + tokens = [] + + for j in range(n_tokens): + token_data = self._lib.whisper_full_get_token_data(ctypes.c_void_p(self.context), i, j) + + tokens.append({ + "id": token_data.id, + "prob": token_data.p, + "logprob": token_data.plog, + "pt": token_data.pt, + "pt_sum": token_data.ptsum, + }) + + segments.append({ + "start": t0, + "end": t1, + "text": txt, + "tokens": tokens, + }) + + result = {"text": all_text.strip(), "text_chunks": text_chunks, "segments": segments} + + return result + + def __dealloc__(self): + # free the memory + self._lib.whisper_free(ctypes.c_void_p(self.context)) + + def unload_model(self): + + self._lib = None + + @staticmethod + def _format_time(t): + + """ Helper utility that formats the time. """ + + msec = t * 10 + hr = msec / (1000 * 60 * 60) + msec = msec - hr * (1000 * 60 * 60) + minu = msec / (1000 * 60) + msec = msec - minu * (1000 * 60) + sec = msec / 1000 + msec = msec - sec * 1000 + + return f'{int(hr):02}:{int(minu):02}:{int(sec):02}.{int(msec):03}' + + def abort_call_back(self, data): + do_nothing = 0 + + def callback(self, ctx, state, i, p): + do_nothing = 0 + + def add_ctypes_configs(self): + + self._lib.whisper_init_from_file.argtypes = [ctypes.c_char_p] + self._lib.whisper_init_from_file.restype = ctypes.c_void_p + + self._lib.whisper_full_default_params.argtypes = [ctypes.c_int] + + if not self.whisper_use_legacy_mac: + self._lib.whisper_full_default_params.restype = whisper_full_params + else: + self._lib.whisper_full_default_params.restype = whisper_full_params_legacy + + if not self.whisper_use_legacy_mac: + self._lib.whisper_full.argtypes = [ctypes.c_void_p, whisper_full_params, ctypes.POINTER(ctypes.c_float), ctypes.c_int] + else: + self._lib.whisper_full.argtypes = [ctypes.c_void_p, whisper_full_params_legacy, ctypes.POINTER(ctypes.c_float), ctypes.c_int] + + self._lib.whisper_full.restype = ctypes.c_int + + self._lib.whisper_full_n_segments.argtypes = [ctypes.c_void_p] + self._lib.whisper_full_n_segments.restype = ctypes.c_int + + self._lib.whisper_full_get_segment_t0.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_get_segment_t0.restype = ctypes.c_int64 + + self._lib.whisper_full_get_segment_t1.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_get_segment_t1.restype = ctypes.c_int64 + + self._lib.whisper_full_get_segment_text.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_get_segment_text.restype = ctypes.c_char_p + + self._lib.whisper_full_n_tokens.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_n_tokens.restype = ctypes.c_int + + self._lib.whisper_full_get_segment_t0.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_get_segment_t0.restype = ctypes.c_int64 + + self._lib.whisper_full_get_segment_t1.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_get_segment_t1.restype = ctypes.c_int64 + + self._lib.whisper_full_get_token_data.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int] + self._lib.whisper_full_get_token_data.restype = whisper_token_data + + self._lib.whisper_full_n_segments.argtypes = [ctypes.c_void_p] + self._lib.whisper_full_n_segments.restype = ctypes.c_int + + self._lib.whisper_full_get_segment_text.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_get_segment_text.restype = ctypes.c_char_p + + self._lib.whisper_full_n_tokens.argtypes = [ctypes.c_void_p, ctypes.c_int] + self._lib.whisper_full_n_tokens.restype = ctypes.c_int + + self._lib.whisper_full_get_token_data.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_int] + self._lib.whisper_full_get_token_data.restype = whisper_token_data + + self._lib.whisper_free.argtypes = [ctypes.c_void_p] + self._lib.whisper_free.restype = None + + self._lib.whisper_log_set.artypes = [ctypes.c_void_p, ctypes.c_void_p] + self._lib.whisper_log_set.restype = None + + return self._lib + + +class LLMWareSemanticModel(BaseModel): + + """ LLMWareSemanticModel class implements the LLMWareSemanticModel API, which is based on the SentenceTransformer + architecture. """ + + def __init__(self, model_name=None, model=None, embedding_dims=None, max_len=150, + model_card=None, api_key=None, **kwargs): + + super().__init__(**kwargs) + + self.model_name = model_name + self.error_message = "\nUnable to process LLMWare Semantic Model. Please try again later" + + self.max_input_len = 512 + self.max_output_len = 512 + self.max_len = max_len + + # to be applied to 'passed-in' Sentence Transformers model + self.normalize_embeddings = True + self.received_loaded_model = False + + # need to parameterize the embedding dims based on model config + if not embedding_dims: + self.embedding_dims = 768 + if model_name == 'mini-lm-sbert': + self.embedding_dims = 384 + + else: + self.embedding_dims = embedding_dims + + self.model_repo_location = LLMWareConfig.get_model_repo_path() + self.model_size="standard" + if model_name == 'mini-lm-sbert': + self.model_size = "mini" + self.transformer_base_model = None + self.sentence = None + + if model: + logger.info("update: SemanticEmbedding model received model - will attempt to load as " + "Sentence Transformer model") + + self.model = model + self.received_loaded_model = True + + if len(model) >= 2: + + try: + # general case is that embedding dimension is the "word_embedding_dimension" of the + # 'Pooling' layer, which is generally the second and last layer of the sbert model + self.embedding_dims = model[1].word_embedding_dimension + + # there are at least 2 edge cases, in which a "Dense" layer is attached after the + # Pooling layer, and further consolidates the embeddings + + if len(model) > 2: + logger.info("update: Sentence Transformer model with more than two layers - unusual - " + " depending upon the architecture, there may be issues loading the model- %s", + len(model)) + + # note: the most common case is with a Dense 3rd layer that maps the Pooling output to + # a different dimension - in this case - this should give the dimensions: + # + # last_layer_config = model[-1].get_config_dict() + # if "out_features" in last_layer_config: + # self.embedding_dims = last_layer_config["out_features"] + + except: + logger.error("error: could not identify model to run embedding - ", model_name) + raise ModelNotFoundException(model_name) + + if model_card and not model: + + if "model_location" in model_card: + if model_card["model_location"] == "st_repo": + # try to pull the model and instantiate directly from Sentence Transformers + try: + from sentence_transformers import SentenceTransformer + except: + raise DependencyNotInstalledException("sentence_transformer") + + try: + self.model = SentenceTransformer(model_card["model_name"]) + except: + raise ModelNotFoundException(model_card["model_name"]) + + if "embedding_dims" in model_card: + self.embedding_dims = model_card["embedding_dims"] + else: + self.embedding_dims = self.model[1].word_embedding_dimension + + def load_model_for_inference(self,fp=None, model_card=None, **kwargs): + + """ This path has been deprecated starting with llmware 0.2.12. """ + + # if fp: self.model_repo_location = fp + + raise LLMWareException(message="Exception - this load option has been deprecated. LLMWareSemanticModels " + "should be pulled from a sentence transformer standard repository.") + + def embedding(self, sentence): + + self.sentence = sentence + + # preview before creating embedding + self.preview() + + # embedding = self.model.encode(sentence, convert_to_tensor=True) + embedding = self.model.encode(self.sentence) + + # add normalization for imported sentence transformer models + """ + if self.received_loaded_model and self.normalize_embeddings: + # normalize embeddings + embedding = torch.tensor(embedding).squeeze(0) + embedding = torch.nn.functional.normalize(embedding, p=2, dim=1) + embedding = embedding.detach().numpy() + """ + + # embedding_2d = embedding.unsqueeze(0) + return embedding + + def cosine_similarity(self, a, b): + return np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)) + + def euclidean_distance(self,a,b): + # aligning with FAISS - which returns square of Euclidean distance + return np.linalg.norm(a - b) * np.linalg.norm(a-b) + + +class ModelResources: + + """ ModelResources is a global state mechanism used in conjunction with deploying the LLMWare Inference + Server class. It manages the persistent loading of multiple models behind the server. """ + + class _ModelState: + models_loaded = 0 + models_list = [] + + @classmethod + def load_model(cls, model_name, sample=False, temperature=0.0, get_logits=True, max_output=200, api_key=None, + use_gpu=True): + + model_card = ModelCatalog().lookup_model_card(model_name) + + if model_card and model_name not in cls._ModelState.models_list: + + setattr(cls._ModelState, model_name, ModelCatalog().load_model(model_name, api_key=api_key, + sample=sample, use_gpu=use_gpu, + get_logits=get_logits, max_output=max_output, + temperature=temperature)) + + cls._ModelState.models_list.append(model_name) + cls._ModelState.models_loaded += 1 + + logger.info(f"update: ModelResources - {cls._ModelState.models_loaded} - " + f"{cls._ModelState.models_list}") + + @classmethod + def unload_model(cls, model_name): + """ Not implemented currently. """ + return 0 + + @classmethod + def check_if_model_loaded(cls, model_name): + + """ Utility method that checks if the model has already been loaded. """ + + if model_name in cls._ModelState.models_list: + return True + return False + + @classmethod + def fetch_model(cls, model_name): + + """ Returns the instantiated model that is already loaded in memory. """ + + return getattr(cls._ModelState, model_name) + + +class MultiModalModel: + """A class to handle multi-modal models, supporting text, image, and other data types.""" + + def __init__(self, model_name, model_type, preprocessors=None, postprocessors=None): + self.model_name = model_name + self.model_type = model_type + self.preprocessors = preprocessors or {} + self.postprocessors = postprocessors or {} + + def add_preprocessor(self, data_type, preprocessor): + """Add a preprocessor for a specific data type.""" + self.preprocessors[data_type] = preprocessor + + def add_postprocessor(self, data_type, postprocessor): + """Add a postprocessor for a specific data type.""" + self.postprocessors[data_type] = postprocessor + + def preprocess(self, data_type, data): + """Preprocess data based on its type.""" + if data_type in self.preprocessors: + return self.preprocessors[data_type](data) + return data + + def postprocess(self, data_type, data): + """Postprocess data based on its type.""" + if data_type in self.postprocessors: + return self.postprocessors[data_type](data) + return data + + def inference(self, inputs): + """Perform inference on multi-modal inputs.""" + processed_inputs = { + data_type: self.preprocess(data_type, data) + for data_type, data in inputs.items() + } + # Placeholder for model inference logic + raw_outputs = self._run_model(processed_inputs) + return { + data_type: self.postprocess(data_type, output) + for data_type, output in raw_outputs.items() + } + + def _run_model(self, inputs): + """Run the model on preprocessed inputs based on the model type.""" + if not hasattr(self, 'model') or self.model is None: + raise ValueError("Model is not loaded. Please load a model before running inference.") + + if self.model_type == "pytorch": + # PyTorch inference + import torch + input_tensors = { + data_type: torch.tensor(data) if isinstance(data, list) else torch.from_numpy(data) + for data_type, data in inputs.items() + } + with torch.no_grad(): + outputs = { + data_type: self.model(input_tensor.unsqueeze(0)) + for data_type, input_tensor in input_tensors.items() + } + return {data_type: output.squeeze(0).numpy() for data_type, output in outputs.items()} + + elif self.model_type == "onnx": + # ONNX inference + import onnxruntime as ort + session = ort.InferenceSession(self.model) + outputs = { + data_type: session.run(None, {session.get_inputs()[0].name: data})[0] + for data_type, data in inputs.items() + } + return outputs + + elif self.model_type == "openvino": + # OpenVino inference + from openvino.runtime import Core + core = Core() + compiled_model = core.compile_model(self.model, "CPU") + outputs = { + data_type: compiled_model([data])[0] + for data_type, data in inputs.items() + } + return outputs + + elif self.model_type == "gguf": + # GGUF inference (example placeholder) + # Assuming GGUF uses a specific library for inference + from llmware.gguf_configs import GGUFInference + gguf_inference = GGUFInference(self.model) + outputs = { + data_type: gguf_inference.run(data) + for data_type, data in inputs.items() + } + return outputs + + elif self.model_type == "tensorflow": + # TensorFlow inference + import tensorflow as tf + input_tensors = { + data_type: tf.convert_to_tensor(data) if isinstance(data, list) else tf.constant(data) + for data_type, data in inputs.items() + } + outputs = { + data_type: self.model(input_tensor[None, ...]) + for data_type, input_tensor in input_tensors.items() + } + return {data_type: output.numpy() for data_type, output in outputs.items()} + + else: + raise ValueError(f"Unsupported model type: {self.model_type}") + +class PyTorchLoader: + + """ PyTorchLoader is a wrapper class that consolidates all of the PyTorch model loading functions + throughout llmware - and provides the ability to create a single custom loader function to over-ride + the default PyTorch model loading, which relies upon HuggingFace repositories, and the formalisms + provided by the transformers library in terms of configs and model class code. This also enables a single + point to customize the behavior of transformers configurations. """ + + def __init__(self, api_key=None, trust_remote_code=True,custom_loader=None): + + self.model_name = None + self.api_key=api_key + self.trust_remote_code = trust_remote_code + self.custom_loader = custom_loader + + def get_generative_model(self, model_name, **kwargs): + + """ Retrieves and instantiates a Pytorch Generative model. Takes a model_name as input, which is + assumed to map to the Huggingface repository name - this name is not necessarily the same as the + LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here + should be the hf_repo attribute on the model card. """ + + # will return None if no model found + model = None + + self.model_name=model_name + + if self.custom_loader: + model = self.custom_loader.loader(self.model_name, + self.api_key,self.trust_remote_code,caller="generative_model",**kwargs) + + else: + + try: + # will wrap in Exception if import fails + from transformers import AutoModelForCausalLM, AutoTokenizer + except ImportError: + raise DependencyNotInstalledException("transformers") + + # insert dynamic pytorch load here + global GLOBAL_TORCH_IMPORT + if not GLOBAL_TORCH_IMPORT: + + logger.debug("Pytorch loader - local dynamic load of torch here") + if util.find_spec("torch"): + + try: + global torch + torch = importlib.import_module("torch") + GLOBAL_TORCH_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load torch module.") + + else: + raise LLMWareException(message="Exception: need to import torch to use this class.") + + if self.api_key: + + if torch.cuda.is_available(): + model = AutoModelForCausalLM.from_pretrained(model_name, token=self.api_key, + trust_remote_code=self.trust_remote_code, + torch_dtype="auto") + else: + model = AutoModelForCausalLM.from_pretrained(model_name, token=self.api_key, + trust_remote_code=self.trust_remote_code) + + else: + if torch.cuda.is_available(): + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=self.trust_remote_code, + torch_dtype="auto") + else: + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=self.trust_remote_code) + + return model + + def get_embedding_model(self, model_name, **kwargs): + + """ Retrieves and instantiates a Pytorch Embedding model. Takes a model_name as input, which is + assumed to map to the Huggingface repository name - this name is not necessarily the same as the + LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here + should be the hf_repo attribute on the model card. """ + + model = None + + self.model_name = model_name + + if self.custom_loader: + model = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code, self.custom_loader, + caller="embedding_model", **kwargs) + + else: + + try: + # will wrap in Exception if import fails + from transformers import AutoModel + except ImportError: + raise DependencyNotInstalledException("transformers") + + # insert dynamic pytorch load here + global GLOBAL_TORCH_IMPORT + if not GLOBAL_TORCH_IMPORT: + + logger.debug("Pytorch loader - local dynamic load of torch here") + if util.find_spec("torch"): + + try: + global torch + torch = importlib.import_module("torch") + GLOBAL_TORCH_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load torch module.") + + else: + raise LLMWareException(message="Exception: need to import torch to use this class.") + + if self.api_key: + + if torch.cuda.is_available(): + model = AutoModel.from_pretrained(model_name, token=self.api_key, + trust_remote_code=self.trust_remote_code, + torch_dtype="auto") + else: + model = AutoModel.from_pretrained(model_name, token=self.api_key, + trust_remote_code=self.trust_remote_code) + + else: + if torch.cuda.is_available(): + model = AutoModel.from_pretrained(model_name, trust_remote_code=self.trust_remote_code, + torch_dtype="auto") + else: + model = AutoModel.from_pretrained(model_name, trust_remote_code=self.trust_remote_code) + + return model + + def get_reranker_model(self, model_name, **kwargs): + + """ Retrieves and instantiates a Pytorch Reranker model. Takes a model_name as input, which is + assumed to map to the Huggingface repository name - this name is not necessarily the same as the + LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here + should be the hf_repo attribute on the model card. """ + + model = None + + self.model_name = model_name + + if self.custom_loader: + model = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code, self.custom_loader, + caller="reranker_model", **kwargs) + + else: + + try: + # will wrap in Exception if import fails + from transformers import AutoModelForSequenceClassification + except ImportError: + raise DependencyNotInstalledException("transformers") + + # insert dynamic pytorch load here + global GLOBAL_TORCH_IMPORT + if not GLOBAL_TORCH_IMPORT: + + logger.debug("Pytorch loader - local dynamic load of torch here") + if util.find_spec("torch"): + + try: + global torch + torch = importlib.import_module("torch") + GLOBAL_TORCH_IMPORT = True + except: + raise LLMWareException(message="Exception: could not load torch module.") + + else: + raise LLMWareException(message="Exception: need to import torch to use this class.") + + if self.api_key: + + if torch.cuda.is_available(): + model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=1, + token=self.api_key, + trust_remote_code=self.trust_remote_code, + torch_dtype="auto") + else: + model = AutoModelForSequenceClassification.from_pretrained(model_name, + num_labels=1, + token=self.api_key, + trust_remote_code=self.trust_remote_code) + + else: + if torch.cuda.is_available(): + model = AutoModelForSequenceClassification.from_pretrained(model_name, + num_labels=1, + trust_remote_code=self.trust_remote_code, + torch_dtype="auto") + else: + model = AutoModelForSequenceClassification.from_pretrained(model_name, + num_labels=1, + trust_remote_code=self.trust_remote_code) + + return model + + def get_tokenizer(self, model_name, **kwargs): + + """ Retrieves and instantiates a tokenizer. Takes a model_name as input, which is + assumed to map to the Huggingface repository name - this name is not necessarily the same as the + LLMWare model card, which is used to lookup the model in model_configs -> the model_name used here + should be the hf_repo attribute on the model card. """ + + tokenizer = None + + self.model_name = model_name + + if self.custom_loader: + tokenizer = self.custom_loader.loader(self.model_name, self.api_key, self.trust_remote_code, + self.custom_loader, caller="tokenizer", **kwargs) + else: + + try: + # will wrap in Exception if import fails + from transformers import AutoTokenizer + except ImportError: + raise DependencyNotInstalledException("transformers") + + if self.api_key: + tokenizer = AutoTokenizer.from_pretrained(model_name, token=self.api_key, + trust_remote_code=self.trust_remote_code) + else: + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=self.trust_remote_code) + + return tokenizer + + +class CustomPTLoader: + + """ CustomPTLoader is a stub class that demonstrates how to create a custom PT loader method that + can be passed to PyTorchLoader to over-ride the default load from a HuggingFace repository using transformers. """ + + def __init__(self, model_name=None, api_key=None, trust_remote_code=True,caller=None): + + self.model_name = model_name + self.api_key= api_key + self.trust_remote_code = trust_remote_code + self.caller = caller + + def loader(self, model_name,api_key=None, trust_remote_code=True, caller=None): + + self.model_name = model_name + self.api_key= api_key + self.trust_remote_code=trust_remote_code + self.caller = caller + + if self.caller == "generative_model": + return self.load_generative_model() + + if self.caller == "embedding_model": + return self.load_embedding_model() + + if self.caller == "tokenizer": + return self.load_tokenizer() + + def load_generative_model(self): + + """ Stub method to enable a custom loading of a generative PyTorch model. """ + + model=None + return model + + def load_embedding_model(self): + + """ Stub method to enable a custom loading of an embedding PyTorch model. """ + + model=None + return model + + def load_tokenizer(self): + + """ Stub method to enable a custom loading a tokenizer. """ + + tokenizer=None + return tokenizer + + +class WindowsLocalFoundryHandler: + + """ Main handler for interface with Windows Local Foundry integration. + Model inferencing handled by implementation of WindowsLocalFoundryModel, + which subclasses BaseModel and mirrors closely the OpenAIModel class. """ + + def __init__(self): + + self.model_id = "" + self.api_key = "" + self.base_url = None + + def get_manager(self): + + """ Checks if manager instance already created, and if not, creates new one. + This is the single entry point to get access to low level manager. """ + + foundry_mgr = _ModelRegistry().get_foundry_manager() + + if not foundry_mgr: + + try: + from foundry_local import FoundryLocalManager + except: + logger.warning(f"WindowsLocalFoundryHandler - could not " + f"load FoundryLocalManager SDK") + return None + + # optional - check local uri + # from foundry_local.service import get_service_uri + # uri = get_service_uri() + + # create new manager and save in ModelHQ state + foundry_mgr = _ModelRegistry().set_foundry_manager(FoundryLocalManager()) + + if foundry_mgr: + if hasattr(foundry_mgr, "endpoint"): + self.base_url = foundry_mgr.endpoint + + return foundry_mgr + + def activate_catalog(self, activate_status): + + """ Connect with Windows Local Foundry, poll for latest model list + and activate in the LLMWare Model Catalog. """ + + result = True + + mgr = self.get_manager() + if not mgr: + logger.info(f"Service not available - can not activate catalog") + activate_status = False + result = False + + if activate_status: + + if not self.is_server_started(): + self.start_server() + + # get available models + create ext catalog + model_list = self.create_model_catalog_extension() + + for model in model_list: + _ModelRegistry().add_model(model) + mn = model.get("model_name", "") + logger.info(f"WindowsLocalFoundryManager - adding foundry model - {mn}") + + else: + + # remove instance from state + _ModelRegistry().reset_foundry_manager() + + return result + + def test_foundry(self): + + """ Confirm that server has started and is running. """ + + mgr = self.get_manager() + + if not mgr: + explanation = ("LocalFoundry Manager could not be created - " + "service does not appear to be available.") + return False, explanation + + started = self.is_server_started() + + if started: + return True, "Server has started" + + else: + # not started + pass + + if mgr: + if hasattr(mgr, "endpoint"): + self.base_url = mgr.endpoint + if hasattr(mgr, "api_key"): + self.api_key = mgr.api_key + + return True, "Server Available but not Started" + + def start_server_if_needed(self): + + """ Start Windows Local Foundry server, if needed. """ + + if not self.is_server_started(): + self.start_server() + + return True + + def is_server_started(self): + + """ Check if Windows Local Foundry server has been started. """ + + mgr = self.get_manager() + started = False + if mgr: + started = mgr.is_service_running() + return started + + def start_server(self): + + """ Start Windows Local Foundry server. """ + + mgr = self.get_manager() + x = mgr.start_service() + return True + + def stop_server(self): + + """ Stop Windows Local Foundry server. """ + + import subprocess + cmd_args = "foundry service stop" + + try: + subprocess.Popen(cmd_args, shell=True) + logger.info(f"WindowsLocalFoundryModel - server " + f"stopped successfully") + + except: + logger.info(f"WindowsLocalFoundryModel - tried to stop server - " + f"unsuccessful - skipping") + + return True + + def check_if_cached(self, model_name): + + """ Check if model is cached in .foundry locally """ + + is_cached = False + + if model_name.endswith("-foundry"): + model_name = model_name[0:-len("-foundry")] + + mgr = self.get_manager() + + if not mgr: + return False + + cached_models = mgr.list_cached_models() + + # check if selected model in cache + for model in cached_models: + + # model_id = model.id + # model_alias = model.alias + if model_name in [model.id, model.alias]: + + is_cached = True + break + + return is_cached + + def download_if_needed(self, model_name): + + """ Download local foundry manager, if not cached """ + + is_cached = self.check_if_cached(model_name) + if not is_cached: + confirmation = self.download_model(model_name) + + return True + + def download_model(self, model_name): + + """ Download model through Windows Local Foundry """ + + # Download and load a model + mgr = self.get_manager() + model_info = mgr.download_model(model_name) + + return model_info + + def load_model(self, model_name,auto_unload=True): + + """ Load model from local .foundry cache """ + + mgr = self.get_manager() + mgr.load_model(model_name) + + return True + + def unload_model(self, model_name): + + """ Unload model from Windows Local Foundry """ + + mgr = self.get_manager() + mgr.unload_model(model_name) + return True + + def list_all_models(self): + + """ List all models available in Foundry Local repository """ + + # List available models in the catalog + mgr = self.get_manager() + catalog = mgr.list_catalog_models() + + # alias | id | version | device_type | runtime | uri | file_size prompt_template | prompt + # device_type - CPU | GPU | NPU + + return catalog + + def release_all_models(self): + + """ Release all models from Windows Local Foundry """ + + # safety check for Windows Local Foundry + + response = False + + if self.is_server_started(): + response = True + mgr = self.get_manager() + list_loaded_models = mgr.list_loaded_models() + if list_loaded_models: + for model in list_loaded_models: + + mgr.unload_model(model.id, force=True) + logger.info(f"release_all_models - unloading model - {model.id}") + + list_loaded_models = mgr.list_loaded_models() + logger.info(f"release_all_models - updated loaded models - " + f"{list_loaded_models}") + + return response + + def _estimate_params(self, file_size_mb): + + """ Quick estimation of the parameter count based on + binary file size of .foundry asset. """ + + # if no indicator found + default_params = 3 + + if file_size_mb >= 7000: + params = 14 + return params + + elif 3000 <= file_size_mb < 7000: + params = 7 + return params + + elif 1000 <= file_size_mb < 3000: + params = 3 + return params + + elif 100 <= file_size_mb < 1000: + params = 1 + return params + + else: + # default - unexpected + return default_params + + def create_model_catalog_extension(self): + + """ Create model catalog entry """ + + mc_ext = [] + models = self.list_all_models() + names_only = [] + + for m in models: + + new_card = {} + ctx = 8192 + + model_mb = m.file_size_mb + params = self._estimate_params(model_mb) + + device_type = "" + if hasattr(m, "device_type"): + device_type = m.device_type + + if device_type not in ["CPU", "GPU", "NPU"]: + if "npu" in m.id: + device_type = "NPU" + elif "gpu" in m.id: + device_type = "GPU" + elif "cpu" in m.id: + device_type = "CPU" + else: + device_type = "CPU" + + if m.id not in names_only: + + new_card.update({"model_name": m.id + "-foundry"}) + + # keep m.id for uniqueness (rather than m.alias) + new_card.update({"display_name": m.id + "-foundry"}) + + new_card.update({"model_family": "WindowsLocalFoundryModel"}) + new_card.update({"model_category": "generative-api"}) + new_card.update({"device_type": device_type}) + new_card.update({"parameters": params}) + new_card.update({"model_location": "api"}) + new_card.update({"context_window": ctx}) + new_card.update({"tags": ["llmware-chat", f"p{params}", + "windows_local_foundry", + "green", "emerald", "api"]}) + + mc_ext.append(new_card) + names_only.append(m.id) + + return mc_ext + + def list_all_cached_models(self): + + """ List all locally cached models in .foundry """ + + model_list = [] + # List models in cache + mgr = self.get_manager() + local_models = mgr.list_cached_models() + logger.info(f"Models in cache - {local_models}") + return model_list + + +class WindowsLocalFoundryModel(BaseModel): + + """ WindowsLocalFoundryModel class implements the Windows Local Foundry API. """ + + def __init__(self, model_name=None, api_key=None, context_window=8192, + max_output=1000, temperature=0.0, **kwargs): + + super().__init__(**kwargs) + + logger.info(f"WindowsLocalFoundryModel - constructing model - {model_name}") + + self.model_class = "WindowsLocalFoundryModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + + # strip "-foundry" identifier + if model_name.endswith("-foundry"): + model_name = model_name[0:-len("-foundry")] + + self.model_name = model_name + + if api_key: + self.api_key = api_key + + self.error_message = ("\nUnable to connect to WindowsLocalFoundry Model. " + "Please try again later.") + + self.separator = "\n" + + # assume input (50%) + output (50%) + self.max_total_len = context_window + self.max_input_len = int(context_window * 0.5) + self.llm_max_output_len = int(context_window * 0.5) + + # inference settings + if temperature >= 0.0: + self.temperature = temperature + else: + self.temperature = 0.0 + + self.target_requested_output_tokens = max_output + self.add_prompt_engineering = False + self.add_context = "" + self.prompt = "" + self.context = "" + + self.instruction_following = False + self.prompt_wrapper = None + + # provides option to pass custom openai_client to + # model class at inference time + self.openai_client = None + + if "model_card" in kwargs: + self.model_card = kwargs["model_card"] + else: + self.model_card = {} + + self.available = True + + self.manager = None + self.base_url = "" + self.api_key = "" + self.model_id = "" + + self.prepare_foundry_manager_and_model() + + self.post_init() + + logger.info(f"WindowsLocalFoundryModel - constructed successfully") + + def prepare_foundry_manager_and_model(self): + + """ Consolidates all init steps around the foundry manager and model """ + + foundry_handler = WindowsLocalFoundryHandler() + + mgr = foundry_handler.get_manager() + list_loaded_models = mgr.list_loaded_models() + + loaded_model = None + + if list_loaded_models: + for model in list_loaded_models: + mgr.unload_model(model.id, force=True) + + logger.info(f"prepare_foundry_manager_and_model - unloading model - {model.id}") + + list_loaded_models = mgr.list_loaded_models() + + logger.info(f"prepare_foundry_manager_and_model - " + f"unloading model - {list_loaded_models}") + + if loaded_model: + if loaded_model != self.model_name: + if mgr: + foundry_handler.unload_model(loaded_model) + + if mgr: + self.available = True + foundry_handler.start_server_if_needed() + confirmation = foundry_handler.download_if_needed(self.model_name) + + mgr.load_model(self.model_name) + + self.manager = mgr + self.base_url = self.manager.endpoint + self.api_key = self.manager.api_key + self.model_id = self.manager.get_model_info(self.model_name).id + + return True + + def prompt_engineer_chatgpt3(self, query, context, inference_dict=None): + + """ Builds prompt in ChatGPT format. """ + + if not self.add_prompt_engineering: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + else: + if context: + selected_prompt = "default_with_context" + else: + selected_prompt = "default_no_context" + + prompt_dict = PromptCatalog().build_core_prompt(prompt_name=selected_prompt, + separator=self.separator, + query=query, context=context, + inference_dict=inference_dict) + + system_message = prompt_dict["prompt_card"]["system_message"] + if not system_message: + system_message = "You are a helpful assistant." + + system_instruction = None + if inference_dict: + if "system_instruction" in inference_dict: + system_instruction = inference_dict["system_instruction"] + if not system_instruction: + system_instruction = system_message + + core_prompt = prompt_dict["core_prompt"] + + messages = [ + {"role": "system", "content": system_instruction}, + {"role": "user", "content": core_prompt} + ] + + return messages + + def prompt_engineer(self, query, context,inference_dict=None): + + # unpack system instruction and chat history + messages = [] + + # this is the core message = context + query + if context: + output = context + "\n" + query + else: + output = query + + chat_history = [] + system_instruction = "" + if inference_dict: + if "chat_history" in inference_dict: + chat_history = inference_dict["chat_history"] + if "system_instruction" in inference_dict: + system_instruction = inference_dict["system_instruction"] + + if not system_instruction: + system_instruction = "You are a helpful assistant." + + # start with system message + messages.append({"role": "system", "content": system_instruction}) + + if chat_history: + for turn in chat_history: + messages.append({"role": "user", "content": turn["user"]}) + messages.append({"role": "assistant", + "content": turn["assistant"]}) + + messages.append({"role": "user", "content": output}) + + return messages + + def load_model_for_inference(self): + + """ Check if model available, and if not load """ + + confirmation = WindowsLocalFoundryHandler().download_if_needed(self.model_name) + + return True + + def unload_model(self, model_name): + + foundry_name = model_name + + if model_name.endswith("-foundry"): + foundry_name = model_name[0:-len("-foundry")] + + try: + from foundry_local import FoundryLocalManager + response = FoundryLocalManager().unload_model(foundry_name, force=True) + logger.info(f"WindowsLocalFoundryModel - " + f"successful unload model") + + except: + logger.info(f"WindowsLocalFoundryModel - unload not successful - " + f"skipping") + + return True + + def close(self): + + logger.info(f"WindowsLocalFoundryModel close model - {self.model_name}") + + foundry_name = self.model_name + + if self.model_name.endswith("-foundry"): + foundry_name = self.model_name[0:-len("-foundry")] + + try: + response = self.manager.unload_model(foundry_name, force=True) + logger.info(f"WindowsLocalFoundryModel - " + f"successful unload model") + except: + logger.info(f"WindowsLocalFoundryModel - unload not successful - " + f"skipping") + + return True + + def inference(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes inference on OpenAI Model. Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. """ + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if "openai_client" in inference_dict: + self.openai_client = inference_dict["openai_client"] + + from llmware.configs import OpenAIConfig + + # call to preview hook (not implemented by default) + self.preview() + + # start change here + + prompt_enriched = self.prompt_engineer(prompt,add_context, + inference_dict=inference_dict) + + # new - change with openai v1 api + try: + from openai import OpenAI + except ImportError: + raise DependencyNotInstalledException("openai >= 1.0") + + usage = {} + time_start = time.time() + + # Configure the client to use the local Foundry service + client = OpenAI(base_url=self.base_url, api_key=self.api_key) + + if self.model_name.endswith("-foundry"): + model_name = self.model_name[0:-(len("-foundry"))] + else: + model_name = self.model_name + + # start here + + # Set the model to use and generate a streaming response + stream = client.chat.completions.create( + model=self.model_id, + # messages=[{"role": "user", "content": prompt_enriched}], + messages=prompt_enriched, + stream=True, + max_tokens=self.target_requested_output_tokens + ) + + text_out = "" + prompt_tokens = 0 + completion_tokens = 0 + + # Print the streaming response + for chunk in stream: + if chunk.choices[0].delta.content is not None: + token = chunk.choices[0].delta.content or "" + # print(chunk.choices[0].delta.content, end="", flush=True) + text_out += token + # yield token + + output_response = {"llm_response": text_out, "usage": usage} + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + def stream(self, prompt, add_context=None, add_prompt_engineering=None, inference_dict=None, + api_key=None): + + """ Executes stream inference on Windows Local Foundry Model with + OpenAI-compatible API. + + Only required input is text-based prompt, with optional + parameters to "add_context" passage that will be assembled using the prompt style in the + "add_prompt_engineering" parameter. Optional inference_dict for temperature and max_tokens configuration, + and optional passing of api_key at time of inference. + """ + + self.available = True + + if not self.available: + logger.warning(f"WindowsLocalFoundryModel - could not connect to service - " + f"unfortunately, model is not available.") + + usage = {"input": 0, "output": 0, "total": 0, "metric": "tokens", + "processing_time": 0.0} + + output_response = {"llm_response": "Service Not Available", + "usage": usage} + + return output_response + + self.prompt = prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + if "openai_client" in inference_dict: + self.openai_client = inference_dict["openai_client"] + + from llmware.configs import OpenAIConfig + + # call to preview hook (not implemented by default) + self.preview() + + # default case - pass the prompt received without change + # prompt_enriched = self.prompt + + prompt_enriched = self.prompt_engineer(prompt,add_context, + inference_dict=inference_dict) + + logger.info(f"WindowsLocalFoundryModel - stream - created prompt - " + f"starting stream") + + # new - change with openai v1 api + try: + from openai import OpenAI + except ImportError: + raise DependencyNotInstalledException("openai >= 1.0") + + usage = {} + time_start = time.time() + + # Configure the client to use the local Foundry service + client = OpenAI(base_url=self.base_url, api_key=self.api_key) + + # Set the model to use and generate a streaming response + stream = client.chat.completions.create( + model=self.model_id, + # messages=[{"role": "user", "content": prompt_enriched}], + messages=prompt_enriched, + stream=True, + max_tokens=self.target_requested_output_tokens + ) + + text_out = "" + prompt_tokens = 0 + completion_tokens = 0 + + # Print the streaming response + for chunk in stream: + if chunk.choices[0].delta.content is not None: + token = chunk.choices[0].delta.content or "" + # print(chunk.choices[0].delta.content, end="", flush=True) + text_out += token + yield token + + usage = {"input": prompt_tokens, + "output": completion_tokens, + "total": prompt_tokens + completion_tokens, + "metric": "tokens", + "processing_time": time.time() - time_start} + + output_response = {"llm_response": text_out, "usage": usage} + + # output inference parameters + self.llm_response = text_out + self.usage = usage + self.logits = None + self.output_tokens = None + self.final_prompt = prompt_enriched + + self.register() + + return output_response + + +class ONNXEmbeddingModel(BaseModel): + + """ ONNXEmbeddingModel class implements support for onnxruntime reranking, + and classifier models. Despite the name, true batch 'embedding' method is + not yet implemented but is on roadmap. + + This is intended to be a simple interface to use encoder-based models in ONNX, + especially for on-device use. """ + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, + model_card=None, embedding_dims=None, max_len=None, + device="CPU", **kwargs): + + super().__init__(**kwargs) + + self.model_class = "ONNXEmbeddingModel" + self.model_category = "embedding" + self.model_name = model_name + self.model = model + self.tokenizer = tokenizer + self.embedding_dims = embedding_dims + self.model_type = None + self.max_total_len = 512 + self.model_architecture = None + self.model_card = model_card + self.safe_buffer = 12 + self.device = device + self.context_window = 512 + + # main handler for model inference session + self.ort_session = None + + if self.model_card: + if "embedding_dims" in self.model_card: + self.embedding_dims = self.model_card["embedding_dims"] + + if "context_window" in self.model_card: + self.context_window = self.model_card["context_window"] + + self.use_gpu = False + self.api_key = api_key + + if self.context_window > self.safe_buffer: + self.max_len = self.context_window - self.safe_buffer + else: + self.max_len = self.context_window + + if max_len: + if max_len: + if max_len < self.context_window: + self.max_len = max_len + + self.text_sample = None + self.model_folder_path = None + + global GLOBAL_ONNX_CORE_RUNTIME + + if not GLOBAL_ONNX_CORE_RUNTIME: + + if util.find_spec("onnxruntime"): + + # note: we import the pybind11 c++ wrapper interface directly + + try: + global ort + ort = importlib.import_module("onnxruntime.capi.onnxruntime_pybind11_state") + GLOBAL_ONNX_CORE_RUNTIME = True + except: + raise LLMWareException(message="ONNXEmbeddingModel: could not load onnxruntime module. " + "If you have pip installed the library, then please check " + "that your platform is supported by onnxruntime.") + + else: + + raise LLMWareException(message="ONNXEmbeddingModel: need to import " + "onnxruntime to use this class, e.g., 'pip3 install " + "onnxruntime`") + + # end dynamic import here + + # self.post_init() + + def load_model_for_inference(self, loading_directions, model_card=None): + + """ Instantiates and loads model from local cache. """ + + if model_card: + self.model_card = model_card + + # onnx expects a string path + self.model_folder_path = loading_directions + + # instantiate the tokenizer from tokenizer.json file + # using hf tokenizers library + from tokenizers import Tokenizer + tokenizer_fn = "tokenizer.json" + self.tokenizer = Tokenizer.from_file(os.path.join(loading_directions, tokenizer_fn)) + + # currently hard-coded - adjust settings to increase size of text + self.tokenizer.enable_padding(length=150) + self.tokenizer.enable_truncation(150) + + # currently hard-coded - load model.onnx file + model_fn = "model.onnx" + onnx_model_path = os.path.join(loading_directions, model_fn) + + # create and initialize InferenceSession in onnxruntime + # -- calling methods directly in the pybind c++ .pyd file + + session_options = ort.get_default_session_options() + self.ort_session = ort.InferenceSession(session_options, onnx_model_path, True, False) + + # TODO: add more options and configs around providers and provider options + providers = [] + provider_options = [dict()] + disabled_optimizers = set() + + self.ort_session.initialize_session(providers, provider_options, disabled_optimizers) + + # end - created and initialized onnxruntime session + + return self + + @staticmethod + def sigmoid(x): + + """ Utility function to return sigmoid """ + + return 1.0 / (1.0 + np.exp(-x)) + + def rank(self, query, text_results, api_key=None, text_index="text", + top_n=20, relevance_threshold=None, min_return=3): + + """ Executes reranking inference. """ + + # call to preview (not implemented by default) + # self.preview() + + batches = [] + if len(text_results) <= 32: + # need to package in chunks + batches.append(text_results) + else: + batch_count = len(text_results) // 32 + if len(text_results) > batch_count * 32: + batch_count += 1 + for x in range(0, batch_count): + stopper = min(len(text_results), (x + 1) * 32) + new_batch = text_results[x * 32:stopper] + batches.append(new_batch) + + output = [] + + for batch in batches: + documents = [] + for i, chunks in enumerate(batch): + documents.append(chunks[text_index]) + + # runs the inference to get similarity score + scores = self.compute_score(query, documents) + + if not isinstance(scores, list): + scores = [scores] + + for i, score in enumerate(scores): + batch[i].update({"rerank_score": score}) + output.append(batch[i]) + + ranked_output = sorted(output, key=lambda x: x["rerank_score"], reverse=True) + + # will return top_n if no relevance threshold set + if not relevance_threshold: + if top_n < len(ranked_output): + final_output = ranked_output[0:top_n] + else: + final_output = ranked_output + else: + final_output = [] + # if relevance threshold, will return all results above threshold + for entries in ranked_output: + if entries["rerank_score"] >= relevance_threshold: + final_output.append(entries) + + # fallback, if no result above threshold, then will return the min number of results + if len(final_output) == 0: + final_output = ranked_output[0:min_return] + + self.register() + + return final_output + + def compute_score(self, query, documents, batch_size: int = 32): + + """ Runs the core ranking inference to determine semantic similarity - + called by rank method """ + + sentence_pairs = [[query, doc] for doc in documents] + + if isinstance(sentence_pairs[0], str): + sentence_pairs = [sentence_pairs] + + self.tokenizer.enable_truncation(100) + self.tokenizer.enable_padding(pad_token="") + + all_scores = [] + for start_index in range(0, len(sentence_pairs), batch_size): + sentence_batch = sentence_pairs[start_index: start_index + batch_size] + + input_ids = [] + attn_mask = [] + + tokenizer_output = self.tokenizer.encode_batch(sentence_batch) + + for sequence in tokenizer_output: + input_ids.append(sequence.ids) + attn_mask.append(sequence.attention_mask) + + input_ids = np.array(input_ids, dtype=np.int64) + attn_mask = np.array(attn_mask, dtype=np.int64) + + # onnxruntime - run inference session + + output_names = [output.name for output in self.ort_session.outputs_meta] + + # replace None with output_names + run_options = None + + output = self.ort_session.run(output_names, {"input_ids": input_ids, + "attention_mask": attn_mask}, run_options) + + # onnxruntime - end run inference session + + scores = self.sigmoid(output[0]) + + if len(documents) == 1: + scores = [scores] + else: + score_float = [] + + # note: convert to 'float' -> safety for json conversion + for score in scores: + if isinstance(score, np.ndarray): + score_float.append(float(score[0])) + else: + score_float.append(float(score)) + + scores = score_float + + all_scores.extend(scores) + + return all_scores + + def classify(self, text, **kwargs): + + """ Executes a classifier inference with ONNX model """ + + config_path = os.path.join(self.model_folder_path, "config.json") + config = None + + if os.path.exists: + try: + config = json.load(open(config_path, "r", errors="ignore")) + except: + logger.warning("onnx classifier config could not be loaded from file") + pass + + if not config: + logger.warning("onnx classifier config - will not be able to convert outputs to keys - no config found.") + + self.tokenizer.enable_truncation(300) + self.tokenizer.enable_padding(pad_token="") + tokenizer_output = self.tokenizer.encode(text) + input_ids = [] + attn_mask = [] + + # for sequence in tokenizer_output: + + input_ids.append(tokenizer_output.ids) + attn_mask.append(tokenizer_output.attention_mask) + + input_ids = np.array(input_ids, dtype=np.int64) + attn_mask = np.array(attn_mask, dtype=np.int64) + + # start here + + output_names = [output.name for output in self.ort_session.outputs_meta] + + # replace None with output_names + run_options = None + + output = self.ort_session.run(output_names, {"input_ids": input_ids, + "attention_mask": attn_mask}, run_options) + + scores = self.sigmoid(output[0]) + + dict_scores = {} + + if config: + dict_scores = [{"label": config["id2label"][str(i)], + "score": score.item()} for i, score in enumerate(scores[0])] + + dict_scores.sort(key=lambda x: x["score"], reverse=True) + + self.register() + + return dict_scores + + +class ONNXVisionGenerativeModel(BaseModel): + + """ONNXVisionGenerativeModel class implements the ONNX generative model API, with + integrated processor, to simplify multi-media processing. Currently this class + supports the phi-3-vision-onnx model by default, and images only. + + Other multimedia types and additional model support - will be added over time. """ + + def __init__(self, model_name=None, api_key=None, model_card=None, + prompt_wrapper=None, instruction_following=False, context_window=2048, + use_gpu_if_available=True, trust_remote_code=True, sample=True, max_output=100, temperature=0.3, + get_logits=False, api_endpoint=None, **kwargs): + + super().__init__() + + self.model_class = "ONNXVisionGenerativeModel" + self.model_category = "generative" + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.final_prompt = None + self.model_name = model_name + self.hf_tokenizer_name = model_name + self.model = None + self.tokenizer = None + self.generator = None + self.sample = sample + self.get_logits = get_logits + self.auto_remediate_function_call_output = True + self.model_card = model_card + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.primary_keys = None + self.function = None + self.fc_supported = False + self.tool_type = None + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + # instantiate if model_name passed without actual model and tokenizer + if model_name and not api_endpoint: + + if not self.model_card: + self.model_card = ModelCatalog().lookup_model_card(self.model_name) + + if self.model_card: + if "hf_repo" in self.model_card: + hf_repo_name = self.model_card["hf_repo"] + self.hf_tokenizer_name = hf_repo_name + + self.model = None + self.tokenizer = None + self.tokenizer_stream = None + + # set to defaults for HF models in Model Catalog + # this can be over-ridden post initiation if needed for custom models + self.prompt_wrapper = "phi_3_vision" + self.instruction_following = False + + # insert dynamic onnx load here + + global GLOBAL_ONNX_GENAI_RUNTIME + + if not GLOBAL_ONNX_GENAI_RUNTIME: + + if util.find_spec("onnxruntime_genai"): + + try: + global og + og = importlib.import_module("onnxruntime_genai") + GLOBAL_ONNX_GENAI_RUNTIME = True + except: + raise LLMWareException(message="ONNXVisionGenerativeModel: could not load onnxruntime_genai module. " + "If you have pip installed the library, then please check " + "that your platform is supported by onnxruntime.") + + else: + import platform + if platform.system() == "Darwin": + raise LLMWareException(message=f"ONNXVisionGenerativeModel: identified current platform as 'Mac OS' " + f"which is not supported for onnxruntime_genai currently. " + f"\nWe would recommend using GGUF for generative inference on a " + f"Mac, or if you wish to use ONNXGenerativeModel, then please " + f"shift to a supported Windows or Linux platform.") + + raise LLMWareException(message="ONNXVisionGenerativeModel: need to import " + "onnxruntime_genai to use this class, e.g., 'pip3 install " + "onnxruntime_genai`") + + # end dynamic import here + + self.params = None + + self.prompt_wrapper = "phi_3_vision" + + if not model_card: + # safety - empty iterable rather than 'None' + model_card = [] + + # deprecated attribute - will be removed in future releases + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + else: + self.instruction_following = False + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + + self.trailing_space = "" + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + + self.model_type = None + self.config = None + + # parameters on context len + output generation + self.max_total_len = context_window + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + + # key output parameters + self.max_output = max_output + self.target_requested_output_tokens = self.max_output + + self.model_architecture = None + self.separator = "\n" + + # use 0 as eos token id by default in generation -> but try to pull from model config + self.eos_token_id = 0 + + self.use_gpu = False + + # no api key expected or required + self.api_key = api_key + + self.error_message = "\nUnable to identify and load HuggingFace model." + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at default of 0.0 + self.temperature = 0.0 + + self.add_prompt_engineering = False + self.add_context = "" + self.context = "" + self.prompt = "" + + self.api_endpoint = api_endpoint + + self.model_repo_path = None + self.model = None + self.processor = None + self.tokenizer_stream = None + + # self.post_init() + + def load_model_for_inference(self, loading_directions, model_card=None): + + """ Loads ONNX Model from local path using loading directions. """ + + self.model_repo_path = loading_directions + + if model_card: + self.model_card = model_card + + self.model = og.Model(loading_directions) + + logger.info("ONNXVisionGenerative Model - constructing model completed.") + + try: + self.processor = self.model.create_multimodal_processor() + except Exception as e: + logger.warning(f"ONNXVisionGenerativeModel - failed to create multimodal " + f"processor with error code: {e}") + return self + + self.tokenizer_stream = self.processor.create_stream() + + return self + + def unload_model(self): + """ Not implemented. """ + return True + + def set_api_key(self, api_key, env_var=""): + """ Not implemented for this model class """ + return True + + def _get_api_key(self, env_var=""): + """ Not implemented for this model class """ + return True + + def inference(self, text_prompt, image_path, **kwargs): + + """ Vision inference expects two inputs - + -- text_prompt: instruction, e.g., 'describe this image' + -- image_path: full file path to supported image type (e.g., jpg, png) + """ + + t0 = time.time() + + if not self.processor: + logger.warning(f"ONNXVisionGenerativeModel - processor not created") + return "" + + image_path = [image_path] + + images = og.Images.open(*image_path) + + # example prompt, e.g., phi-3-vision + # prompt = "<|user|>\n" + "<|image_1|>\n" + text_prompt + "<|end|>\n<|assistant|>\n" + + prompt = PromptCatalog().apply_prompt_wrapper(text_prompt,self.prompt_wrapper) + + try: + inputs = self.processor(prompt, images=images) + except Exception as e: + logger.info(f"ONNXVisionGenerativeModel - processor not successful - " + f"generated run time error - {e}") + inputs = [] + + logging.info("ONNXVisionGenerative Model - Generating response.") + + params = og.GeneratorParams(self.model) + max_tokens = 7680 + params.set_search_options(max_length=max_tokens) + generator = og.Generator(self.model, params) + generator.set_inputs(inputs) + token_count = 0 + output_text = "" + + while not generator.is_done(): + + generator.generate_next_token() + + new_token = generator.get_next_tokens()[0] + new_token_dec = self.tokenizer_stream.decode(new_token) + output_text += new_token_dec + + token_count += 1 + if token_count > max_tokens: + break + + logging.info(f"\nONNXVisionGenerativeModel - tokens generated: {token_count}") + logging.info(f"\nONNXVisionGenerative Model - processing time: {time.time()-t0}") + + t1 = time.time() + + # todo: will add separate counting of input tokens + input_token_count = 0 + + response = {"llm_response": output_text, + "usage": {"input": input_token_count, + "output": token_count, + "total": input_token_count +token_count, + "metric": "tokens", + "processing_time": t1-t0}} + + return response + + def stream(self, text_prompt, image_path, **kwargs): + + """ Vision stream inference expects two inputs - + -- text_prompt: instruction, e.g., 'describe this image' + -- image_path: full file path to supported image type (e.g., jpg, png) + + note: initial image encoding can easily take 10-20 seconds, depending upon system, + and then stream generation output is rapid after that. + + """ + + t0 = time.time() + + if not self.processor: + logger.warning(f"ONNXVisionGenerativeModel - processor not created") + return "" + + image_path = [image_path] + + images = og.Images.open(*image_path) + + # e.g., prompt for phi-3-vision currently + # prompt = "<|user|>\n" + "<|image_1|>\n" + text_prompt + "<|end|>\n<|assistant|>\n" + + prompt = PromptCatalog().apply_prompt_wrapper(text_prompt, self.prompt_wrapper) + + try: + inputs = self.processor(prompt, images=images) + except Exception as e: + logger.info(f"ONNXVisionGenerativeModel - processor not successful - " + f"generated run time error - {e}") + inputs = [] + + logging.info("ONNXVisionGenerative Model - Generating response.") + + params = og.GeneratorParams(self.model) + max_tokens = 7680 + params.set_search_options(max_length=max_tokens) + generator = og.Generator(self.model, params) + generator.set_inputs(inputs) + token_count = 0 + output_text = "" + + while not generator.is_done(): + + generator.generate_next_token() + + new_token = generator.get_next_tokens()[0] + new_token_dec = self.tokenizer_stream.decode(new_token) + output_text += new_token_dec + + token_count += 1 + if token_count > max_tokens: + break + + yield new_token_dec + + logging.info(f"\nONNXVisionGenerativeModel - tokens generated: {token_count}") + logging.info(f"\nONNXVisionGenerative Model - processing time: {time.time()-t0}") + + self.register() + + return output_text + + def cleanup_stream_gen_on_early_stop(self): + self.generator = None + return True + + def register_top_logits(self, logit): + + """ Gets the top logits and keeps a running log for output analysis. """ + + # logit will be in form of (1,1,vocab_len), for all but the first logit + # if first logit (will have shape of context len - add [-1]) + + if logit.shape[1] > 1: + # used for first logit with shape, e.g., (1,input_token_len,vocab_size) + logit_array = logit.squeeze()[-1] + else: + # all other logits after the first token + logit_array = logit.squeeze() + + logit_size = logit.shape[-1] + + # useful check on shape of logit_array + logit_array_size = logit_array.shape + + sm = np.exp(logit_array) / sum(np.exp(logit_array)) + + sm_sorted = np.sort(sm) + sm_args_sorted = np.argsort(sm) + + top_logits = [] + + for x in range(0, self.top_logit_count): + # round the float number to 3 digits + pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3)) + top_logits.append(pair) + + self.logits_record.append(top_logits) + + return top_logits + + +class _OVInfer: + + """ Wrapper to package inputs and outputs in connection with executing a + forward pass on OpenVINO model (e.g., infer_request) - derived closely from + utilities provided in OpenVINO, e.g.: + + https://github.com/openvinotoolkit/openvino/blob/master/src/bindings/python/src/openvino/utils/data_helpers/data_dispatcher.py + + Not intended to be called directly, but is used as utility within other + OV model classes. + """ + + def __init__(self, _infer_request=None): + self._infer_request = _infer_request + + def ov_core_inference(self, inputs, + _infer_request, + share_outputs=False, + decode_strings=True): + + """ Primary entrypoint into _OVInfer - takes the 'raw' inputs and + infer_request instance, and wraps both the inputs, calls the + forward pass on the infer request, and then wraps the outputs. """ + + self._infer_request = _infer_request + + if inputs is None: + inputs = {} + + # by default + is_shared = True + + # prepare model inputs + if is_shared: + model_inputs = self._create_shared(inputs, _infer_request) + else: + model_inputs = self._create_copied(inputs, _infer_request) + + # run inference + response = _infer_request.infer(model_inputs, + share_outputs=share_outputs, + decode_strings=decode_strings) + + # package up response + ov_dict = OVDict(response) + + return ov_dict + + def _create_shared(self, inputs, request): + + if isinstance(inputs, dict) or isinstance(inputs, tuple) or isinstance(inputs, OVDict): + inp_n = self.normalize_arrays(inputs, is_shared=True) + return {k: self.value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in inp_n.items()} + + elif isinstance(inputs, list): + if len(request.input_tensors) == 1: + is_single_input = True + else: + is_single_input = False + + inputs_x = self.normalize_arrays( + [inputs] if is_single_input and self.is_list_simple_type(inputs) else inputs, is_shared=True) + + return {k: self.value_to_tensor(v, request=request, is_shared=True, key=k) for k, v in inputs_x.items()} + + elif isinstance(inputs, np.ndarray): + inp = self.normalize_arrays(inputs, is_shared=True) + return self.value_to_tensor(inp, request=request, is_shared=True) + + elif isinstance(inputs, int) or isinstance(inputs, float) or isinstance(inputs, str) \ + or isinstance(inputs, bytes) or isinstance(inputs, ovc.Tensor) or isinstance(inputs, np.number): + return self.value_to_tensor(inputs, request=request, is_shared=True) + + # Check the special case of the array-interface + if hasattr(inputs, "__array__"): + request._inputs_data = self.normalize_arrays(inputs, is_shared=True) + return self.value_to_tensor(request._inputs_data, request=request, is_shared=True) + + # raise error if incompatible type + raise LLMWareException(message=f"_OVInfer - _created_share - " + f"incompatible inputs of type: {type(inputs)}") + + def _create_copied(self, inputs, request): + + if isinstance(inputs, dict) or isinstance(inputs, tuple) or isinstance(OVDict): + return self.update_inputs(self.normalize_arrays(inputs, is_shared=False), request) + + elif isinstance(inputs, list): + return self.update_inputs( + self.normalize_arrays([inputs] if request._is_single_input() and self.is_list_simple_type(inputs) else inputs, + is_shared=False), request) + + elif isinstance(inputs, np.ndarray): + self.update_tensor(self.normalize_arrays(inputs, is_shared=False), request, key=None) + return {} + + elif isinstance(inputs, ovc.Tensor) or isinstance(inputs, np.number) or isinstance(inputs, int) or \ + isinstance(inputs, float) or isinstance(inputs, str) or isinstance(inputs, bytes): + return self.value_to_tensor(inputs, request=request, is_shared=False) + + # Check the special case of the array-interface + if hasattr(inputs, "__array__"): + self.update_tensor(self.normalize_arrays(inputs, is_shared=False), request, key=None) + return {} + + # raise error if incompatible type + raise LLMWareException(message=f"_OVInfer - _created_copied - " + f"incompatible inputs of type: {type(inputs)}") + + def get_request_tensor(self, request, key=None): + + """ Retrieves the input tensor from a request instance. """ + + if key is None: + return request.get_input_tensor() + elif isinstance(key, int): + return request.get_input_tensor(key) + elif isinstance(key, (str, ovc.ConstOutput)): + return request.get_tensor(key) + else: + raise LLMWareException(message=f"_OVInfer - get_request_tensor - " + f"key type {type(key)} is not " + f"supported for Tensor key: {key}") + + def value_to_tensor(self, value, request=None, is_shared: bool = False, key=None) -> None: + + """ Converts value to OV tensor """ + + if isinstance(value, ovc.Tensor): + return value + + elif isinstance(value, np.ndarray): + tensor = self.get_request_tensor(request, key) + tensor_type = tensor.get_element_type() + tensor_dtype = tensor_type.to_dtype() + # String edge-case, always copy. + # Scalars are also handled by C++. + if tensor_type == ovc.Type.string: + return ovc.Tensor(value, shared_memory=False) + # Scalars edge-case: + if value.ndim == 0: + tensor_shape = tuple(tensor.shape) + if tensor_dtype == value.dtype and tensor_shape == value.shape: + return ovc.Tensor(value, shared_memory=is_shared) + elif tensor.size == 0: + # the first infer request for dynamic input cannot reshape to 0 shape + return ovc.Tensor(value.astype(tensor_dtype).reshape((1)), shared_memory=False) + else: + return ovc.Tensor(value.astype(tensor_dtype).reshape(tensor_shape), shared_memory=False) + # WA for FP16-->BF16 edge-case, always copy. + if tensor_type == ovc.Type.bf16: + tensor = ovc.Tensor(tensor_type, value.shape) + tensor.data[:] = value.view(tensor_dtype) + return tensor + + # WA for "not writeable" edge-case, always copy. + if value.flags["WRITEABLE"] is False: + tensor = ovc.Tensor(tensor_type, value.shape) + tensor.data[:] = value.astype(tensor_dtype) if tensor_dtype != value.dtype else value + return tensor + # If types are mismatched, convert and always copy. + if tensor_dtype != value.dtype: + return ovc.Tensor(value.astype(tensor_dtype), shared_memory=False) + # Otherwise, use mode defined in the call. + return ovc.Tensor(value, shared_memory=is_shared) + + elif isinstance(value, list): + return ovc.Tensor(value) + + elif isinstance(value, int) or isinstance(value, float) or isinstance(value, str) or \ + isinstance(value, bytes) or isinstance(value, np.number): + # np.number/int/float/str/bytes edge-case, copy will occur in both scenarios. + tensor_type = self.get_request_tensor(request, key).get_element_type() + tensor_dtype = tensor_type.to_dtype() + tmp = np.array(value) + # String edge-case -- it converts the data inside of Tensor class. + # If types are mismatched, convert. + if tensor_type != ovc.Type.string and tensor_dtype != tmp.dtype: + return ovc.Tensor(tmp.astype(tensor_dtype), shared_memory=False) + return ovc.Tensor(tmp, shared_memory=False) + + # raise error if incompatible type + raise LLMWareException(message=f"_OVInfer - value_to_tensor - " + f"incompatible inputs of type: {type(value)}") + + def to_c_style(self, value: Any, is_shared: bool = False) -> Any: + + if not isinstance(value, np.ndarray): + if hasattr(value, "__array__"): + return self.to_c_style(np.array(value, copy=False), is_shared) if is_shared else np.array(value, copy=True) + return value + return value if value.flags["C_CONTIGUOUS"] else np.ascontiguousarray(value) + + def normalize_arrays(self, inputs: Any, is_shared: bool = False) -> Any: + + if isinstance(inputs, dict): + return {k: self.to_c_style(v, is_shared) if is_shared else v for k, v in inputs.items()} + + if isinstance(inputs, OVDict): + return {i: self.to_c_style(v, is_shared) if is_shared else v for i, (_, v) in enumerate(inputs.items())} + + if isinstance(inputs, list) or isinstance(inputs, tuple): + return {i: self.to_c_style(v, is_shared) if is_shared else v for i, v in enumerate(inputs)} + + if isinstance(inputs, np.ndarray): + return self.to_c_style(inputs, is_shared) if is_shared else inputs + + # Check the special case of the array-interface + if hasattr(inputs, "__array__"): + return self.to_c_style(np.array(inputs, copy=False), is_shared) if is_shared else np.array(inputs, copy=True) + + # raise error if incompatible type + raise LLMWareException(message=f"_OVInfer - normalize_arrays - " + f"incompatible inputs of type: {type(inputs)}") + + def set_request_tensor(self, request, tensor, key=None) -> None: + + if key is None: + request.set_input_tensor(tensor) + elif isinstance(key, int): + request.set_input_tensor(key, tensor) + elif isinstance(key, (str, ovc.ConstOutput)): + request.set_tensor(key, tensor) + else: + # raise error if incompatible type + raise LLMWareException(message=f"_OVInfer - set_request_tensor - " + f"unsupported key type: {type(key)} for " + f"tensor under key: {key}") + + def update_tensor(self, inputs: Any, request, key=None) -> None: + + if isinstance(inputs, np.ndarray): + if inputs.ndim != 0: + tensor = self.get_request_tensor(request, key) + # Update shape if there is a mismatch + if tuple(tensor.shape) != inputs.shape: + tensor.shape = inputs.shape + # When copying, type should be up/down-casted automatically. + if tensor.element_type == ovc.Type.string: + tensor.bytes_data = inputs + else: + tensor.data[:] = inputs[:] + else: + # If shape is "empty", assume this is a scalar value + self.set_request_tensor( + request, + self.value_to_tensor(inputs, request=request, is_shared=False, key=key), + key, + ) + + # TODO: what to return + + elif isinstance(inputs, np.number) or isinstance(inputs, float) or isinstance(inputs, int) or \ + isinstance(inputs, str): + self.set_request_tensor( + request, + self.value_to_tensor(inputs, request=request, is_shared=False, key=key), + key, + ) + + if hasattr(inputs, "__array__"): + self.update_tensor(self.normalize_arrays(inputs, is_shared=False), request, key) + return None + + # raise error if unsupported key type + raise LLMWareException(message=f"_OVInfer - update_tensor - " + f"unsupported key type: {type(inputs)} for " + f"tensor under key: {key}") + + def update_inputs(self, inputs: dict, request): + + # Create new temporary dictionary. + # new_inputs will be used to transfer data to inference calls, + # ensuring that original inputs are not overwritten with Tensors. + new_inputs = {} + + for key, value in inputs.items(): + if not isinstance(key, (str, int, ovc.ConstOutput)): + raise TypeError(f"Incompatible key type for input: {key}") + # Copy numpy arrays to already allocated Tensors. + # If value object has __array__ attribute, load it to Tensor using np.array + if isinstance(value, (np.ndarray, np.number, int, float, str)) or hasattr(value, "__array__"): + self.update_tensor(value, request, key) + elif isinstance(value, list): + new_inputs[key] = ovc.Tensor(value) + # If value is of Tensor type, put it into temporary dictionary. + elif isinstance(value, ovc.Tensor): + new_inputs[key] = value + # Throw error otherwise. + else: + + # raise error if unsupported type + raise LLMWareException(message=f"_OVInfer - update_inputs - " + f"unsupported key type: {type(value)} for " + f"tensor under key: {key}") + + return new_inputs + + def is_list_simple_type(self, input_list: list) -> bool: + + for sublist in input_list: + if isinstance(sublist, list): + for element in sublist: + if not isinstance(element, (str, float, int, bytes)): + return False + else: + if not isinstance(sublist, (str, float, int, bytes)): + return False + return True + + +class OVDict(Mapping): + + """ Output handler for OV infer request forward pass, used for + downstream processing in OVEmbeddingModel class - mirrors + OpenVINO OVDict definition. """ + + def __init__(self, _dict): + self._dict = _dict + self._names = None + + def __iter__(self): + return self._dict.__iter__() + + def __len__(self) -> int: + return len(self._dict) + + def __repr__(self) -> str: + return self._dict.__repr__() + + def __get_names(self): + return {key: key.get_names() for key in self._dict.keys()} + + def __get_key(self, index: int): + return list(self._dict.keys())[index] + + def __getitem__(self, key) -> np.ndarray: + + if isinstance(key, str): + if self._names is None: + self._names = self.__get_names() + for port, port_names in self._names.items(): + if key in port_names: + return self._dict[port] + raise KeyError(key) + + elif isinstance(key, int): + try: + return self._dict[self.__get_key(key)] + except IndexError: + raise KeyError(key) + else: + try: + return self._dict[key] + except: + raise LLMWareException(message=f"OVDict - unknown key type - {type(key)}") + + def keys(self): + return self._dict.keys() + + def values(self): + return self._dict.values() + + def items(self): + return self._dict.items() + + def names(self): + + if self._names is None: + self._names = self.__get_names() + return tuple(self._names.values()) + + def to_dict(self): + return self._dict + + def to_tuple(self): + return tuple(self._dict.values()) + + +class OVEmbeddingModel(BaseModel): + + """ OVEmbeddingModel class implements a high-level interface to use + OpenVINO encoder-based models, supporting three different modalities currently: + + -- Embedding - for use with vector databases + -- Reranker - for in-memory semantic similarity comparisons + -- Classify - for classifier based models + + """ + + def __init__(self, model=None, tokenizer=None, model_name=None, api_key=None, model_card=None, + embedding_dims=None, use_gpu_if_available=True, max_len=None, device="CPU", **kwargs): + + super().__init__(**kwargs) + + self.model_class = "OVEmbeddingModel" + self.model_category = "embedding" + + self.model_name = model_name + self.model = model + self.tokenizer= tokenizer + self.embedding_dims = embedding_dims + self.model_type = None + self.max_total_len = 512 + self.model_architecture = None + self.model_card = model_card + self.safe_buffer = 12 + self.device = device + + # default for HF embedding model -> will be over-ridden by model card / configs, if available + self.context_window = 512 + + if self.model_card: + if "embedding_dims" in self.model_card: + self.embedding_dims = self.model_card["embedding_dims"] + + if "context_window" in self.model_card: + self.context_window = self.model_card["context_window"] + + if "model_name" in self.model_card: + self.model_name = self.model_card["model_name"] + + global ovc + global GLOBAL_OPENVINO_IMPORT + if not GLOBAL_OPENVINO_IMPORT: + + if not util.find_spec("openvino"): + raise LLMWareException(message="OVEmbeddingModel: to use OVEmbeddingModel requires " + "install of 'openvino' library. " + "Please try: `pip3 install openvino` " + "and confirm that your " + "hardware platform is supported.") + + if util.find_spec("openvino"): + + # loads/accesses the openvino pybind pyd methods directly + + try: + ovc = importlib.import_module("openvino._pyopenvino") + GLOBAL_OPENVINO_IMPORT = True + except: + raise LLMWareException(message="OVEmbeddingModel: could not load openvino module.") + + if not ovc: + raise LLMWareException(message="OVEmbeddingModel: could not load required openvino dependency.") + + # end dynamic import here + + self.use_gpu = False + + # no api key expected or required + self.api_key = api_key + + # set max len for tokenizer truncation with 'safe_buffer' below context_window size + if self.context_window > self.safe_buffer: + self.max_len = self.context_window - self.safe_buffer + else: + self.max_len = self.context_window + + # option to set smaller size than model context window + if max_len: + if max_len < self.context_window: + self.max_len = max_len + + self.text_sample = None + + self.model_folder_path = None + self._device = self.device + self.is_dynamic = True + self.read_model_xml_path = None + self.model = None + self.request = None + self._infer_request = None + self.input_names = None + self.output_names = None + self.config = None + + # post init not implemented for this model class currently + # self.post_init() + + def load_model_for_inference(self, loading_directions, model_card=None): + + """ Loads OV Embedding Model from local path using loading directions. """ + + if model_card: + self.model_card = model_card + + self.model_folder_path = Path(loading_directions) + + # load the tokenizer from tokenizer.json in model repo + from tokenizers import Tokenizer + + tokenizer_fn = "tokenizer.json" + self.tokenizer = Tokenizer.from_file(os.path.join(loading_directions, tokenizer_fn)) + + # hard-coded at 150 tokens -> adjust to increase/decrease + self.tokenizer.enable_padding(length=150) + self.tokenizer.enable_truncation(150) + + if not ovc: + logger.warning("OVEmbeddingModel - could not find backend module") + return False + + # need to get config.json file + self.config = self.get_config_from_file() + + self.read_model_xml_path = Path(os.path.join(loading_directions, "openvino_model.xml")) + + core = ovc.Core() + self.model = core.read_model(self.read_model_xml_path.resolve(), + self.read_model_xml_path.with_suffix(".bin").resolve()) + + if self.is_dynamic: + height = None + width = None + self.model = self._reshape(self.model, -1, -1, height, width) + + input_names = {} + for idx, key in enumerate(self.model.inputs): + names = tuple(key.get_names()) + input_names[next((name for name in names if "/" not in name), names[0])] = idx + self.input_names = input_names + + output_names = {} + for idx, key in enumerate(self.model.outputs): + names = tuple(key.get_names()) + output_names[next((name for name in names if "/" not in name), names[0])] = idx + self.output_names = output_names + + self.request = None + + if self.request is None: + + # try to load on GPU first, and fallback to CPU, if GPU fails + + try: + gpu_device_name = core.get_property("GPU", "FULL_DEVICE_NAME") + logger.info(f"OVGenerativeModel - found gpu device - name: {gpu_device_name}.") + device = "GPU" + logger.info(f"OVEmbeddingModel - successful finding GPU") + + except: + logger.debug("OVGenerativeModel - loading - could not find gpu - setting device for CPU") + device = "CPU" + + self._device = device + logger.info(f"OVEmbeddingModel - device - {device}") + + ov_config = {} + self.request = core.compile_model(self.model, self._device, ov_config) + + logger.info(f"OVEmbedding - completed model compile - {self.model_name} " + f"on device - {self._device}") + + return self + + def get_config_from_file(self): + + """ Loads config information from config.json file """ + + config_file = os.path.join(self.model_folder_path, "config.json") + + try: + config = json.load(open(config_file, "r")) + except: + config = {} + + return config + + def _inference(self, inputs): + + """ Internal inference method implements forward pass on the model """ + + if not self._infer_request: + self._infer_request = self.request.create_infer_request() + + try: + outputs = _OVInfer().ov_core_inference(inputs, + self._infer_request, + share_outputs=False, + decode_strings=True) + + except Exception as e: + raise LLMWareException(message=f"OVEmbeddingModel - _inference - " + f"unsuccessful - generated error code - " + f"{e}") + + return outputs + + def set_api_key(self, api_key, env_var=""): + """ Not implemented """ + return True + + def _get_api_key(self, env_var=""): + """ Not implemented """ + return True + + def token_counter(self, text_sample): + + """ Counts tokens in text sample. """ + + toks = self.tokenizer.encode(text_sample).ids + return len(toks) + + @staticmethod + def sigmoid(x): + """Simple sigmoid function. Not numerically stable!""" + return 1.0 / (1.0 + np.exp(-x)) + + def _reshape(self, model, batch_size, sequence_length, height=None, width=None): + + """ Internal implementation method to reshape the input """ + + shapes = {} + for inputs in model.inputs: + shapes[inputs] = inputs.get_partial_shape() + shapes[inputs][0] = batch_size + shapes[inputs][1] = sequence_length + if height is not None: + shapes[inputs][2] = height + if width is not None: + shapes[inputs][3] = width + model.reshape(shapes) + return model + + def reshape(self, batch_size, sequence_length, height=None, width= None): + + """ Reshape input """ + + self.is_dynamic = True if batch_size == -1 and sequence_length == -1 else False + self.model = self._reshape(self.model, batch_size, sequence_length, height, width) + self.request = None + return self + + def forward(self, input_ids, attention_mask, token_type_ids = None, **kwargs): + + """ Forward pass on model """ + + np_inputs = isinstance(input_ids, np.ndarray) + if not np_inputs: + input_ids = np.array(input_ids) + attention_mask = np.array(attention_mask) + token_type_ids = np.array(token_type_ids) if token_type_ids is not None else token_type_ids + + inputs = { + "input_ids": input_ids, + "attention_mask": attention_mask, + } + + # Add the token_type_ids when needed + if "token_type_ids" in self.input_names: + inputs["token_type_ids"] = token_type_ids if token_type_ids is not None else np.zeros_like(input_ids) + + outputs = self._inference(inputs) + + last_hidden_state = outputs["last_hidden_state"] + + embedding = last_hidden_state[:,0] + + return embedding + + def classify(self, text,**kwargs): + + """ Implements a classify inference for classifier-based models that + have been fine-tuned with a classifier head""" + + self.text_sample = text + + if not isinstance(self.text_sample, list): + self.text_sample = [self.text_sample] + + input_ids = [] + attn_mask = [] + + tokenizer_output = self.tokenizer.encode_batch(self.text_sample) + + for sequence in tokenizer_output: + input_ids.append(sequence.ids) + attn_mask.append(sequence.attention_mask) + + input_ids = np.array(input_ids) + attn_mask = np.array(attn_mask) + + np_inputs = isinstance(input_ids, np.ndarray) + if not np_inputs: + input_ids = np.array(input_ids) + attn_mask = np.array(attn_mask) + + inputs = { + "input_ids": input_ids, + "attention_mask": attn_mask, + } + + # Add the token_type_ids when needed + if "token_type_ids" in self.input_names: + # may require customization for some model types + inputs["token_type_ids"] = np.zeros_like(input_ids) + + outputs = self._inference(inputs) + + logits = outputs["logits"] + + max_value = np.max(logits, axis=-1, keepdims=True) + shifted_exp = np.exp(logits - max_value) + scores = shifted_exp / shifted_exp.sum(axis=-1, keepdims=True) + + if "id2label" in self.config: + try: + dict_scores = [{"label": self.config["id2label"][str(i)], + "score": score.item()} for i, score in enumerate(scores[0])] + except: + dict_scores = [{"label": "NA", "score": 0.0}] + logger.info(f"OVEmbeddingModel - classify configs not resolved - {self.config} - " + f"{scores[0]}") + else: + # report scores without label if not available (e.g, missing config) + dict_scores = [] + for i, score in enumerate(scores[0]): + new_entry = {"label": f"score_{i+1}", "score": score.item()} + dict_scores.append(new_entry) + + dict_scores.sort(key=lambda x: x["score"], reverse=True) + + self.register() + + return dict_scores + + def embedding (self, text_sample, api_key=None): + + """ Executes embedding inference. """ + + self.text_sample = text_sample + + # call to preview (not implemented by default) + # self.preview() + + # return embeddings only + if not isinstance(self.text_sample,list): + self.text_sample = [self.text_sample] + + input_ids = [] + attn_mask = [] + + tokenizer_output = self.tokenizer.encode_batch(self.text_sample) + + for sequence in tokenizer_output: + input_ids.append(sequence.ids) + attn_mask.append(sequence.attention_mask) + + input_ids = np.array(input_ids) + attn_mask = np.array(attn_mask) + + model_input = {"input_ids": input_ids, "attention_mask": attn_mask} + + # Add the token_type_ids when needed + if "token_type_ids" in self.input_names: + model_input["token_type_ids"] = np.zeros_like(input_ids) + + outputs = self._inference(model_input) + + last_hidden_state = outputs["last_hidden_state"] + + embedding = last_hidden_state[:,0] + + # l2 normalization with numpy + embeddings_normalized = embedding / np.linalg.norm(embedding,2,axis=1,keepdims=True) + + self.register() + + return embeddings_normalized + + def rank (self, query, text_results, text_index="text", + api_key=None, top_n=20, relevance_threshold=None, min_return=3): + + """ Executes reranking inference. """ + + # call to preview (not implemented by default) + # self.preview() + + batches = [] + if len(text_results) <= 32: + # need to package in chunks + batches.append(text_results) + else: + batch_count = len(text_results) // 32 + if len(text_results) > batch_count * 32: + batch_count += 1 + for x in range(0,batch_count): + stopper = min(len(text_results), (x+1)*32) + new_batch = text_results[x*32:stopper] + batches.append(new_batch) + + output = [] + + for batch in batches: + documents = [] + for i, chunks in enumerate(batch): + documents.append(chunks[text_index]) + + scores = self.compute_score(query, documents) + + if not isinstance(scores,list): + scores = [scores] + + for i, score in enumerate(scores): + batch[i].update({"rerank_score": score}) + output.append(batch[i]) + + ranked_output = sorted(output, key=lambda x: x["rerank_score"], reverse=True) + + # will return top_n if no relevance threshold set + if not relevance_threshold: + if top_n < len(ranked_output): + final_output = ranked_output[0:top_n] + else: + final_output = ranked_output + else: + final_output = [] + # if relevance threshold, will return all results above threshold + for entries in ranked_output: + if entries["rerank_score"] >= relevance_threshold: + final_output.append(entries) + + # fallback, if no result above threshold, then will return the min number of results + if len(final_output) == 0: + final_output = ranked_output[0:min_return] + + self.register() + + return final_output + + def compute_score(self, query, documents, batch_size: int = 32): + + """ Applies semantic similarity ranker to query and a set of text chunks. """ + + sentence_pairs = [[query, doc] for doc in documents] + + # if empty query, then return [] for empty scores + if len(sentence_pairs) == 0: + return [] + + assert isinstance(sentence_pairs, list) + if isinstance(sentence_pairs[0], str): + sentence_pairs = [sentence_pairs] + + #TODO: look at truncation settings + self.tokenizer.enable_truncation(100) + self.tokenizer.enable_padding(pad_token="") + + all_scores = [] + for start_index in range(0, len(sentence_pairs), batch_size): + sentences_batch = sentence_pairs[start_index: start_index + batch_size] + + input_ids = [] + attn_mask = [] + + tokenizer_output = self.tokenizer.encode_batch(sentences_batch) + + for sequence in tokenizer_output: + input_ids.append(sequence.ids) + attn_mask.append(sequence.attention_mask) + + input_ids = np.array(input_ids) + attn_mask = np.array(attn_mask) + + # note: last element is the 'position_type_ids' + model_input = (input_ids, attn_mask, np.zeros_like(input_ids)) + + scores = self._inference(model_input) + scores = self.sigmoid(scores["logits"].squeeze()) + + # safety check if single value, e.g., if input is only one document + if len(documents) == 1: + scores = [scores] + else: + scores = scores.tolist() + all_scores.extend(scores) + + if len(all_scores) == 1: + all_scores = all_scores[0] + + return all_scores + + +class GGUFVisionGenerativeModel(BaseModel): + + """ Implementation of GGUF Vision Model class - instantiate and run vision-to-text inferences using + GGUF llama.cpp models with MTMD CLIP-based visual encoding - wraps two underlying models (which interact + directly with each other) - + + -- decoder generative model, e.g., llama main + -- encoder clip model, e.g., mtmd clip model + + """ + + def __init__(self, model_name=None, model_card=None, api_key=None, prompt_wrapper=None, instruction_following=False, + context_window=2048, use_gpu_if_available=True, get_logits=False, + sample=True, max_output=500, temperature=0.0, api_endpoint=None, **kwargs): + + super().__init__(**kwargs) + + logger.debug("GGUFVisionGenerativeModel - initializing GGUF Vision model ... ") + + self.model_class = "GGUFVisionGenerativeModel" + self.model_category = "generative" + + # key model state attributes + self.gguf_file = None + self.gguf_repo = None + self.clip_file = None + self.clip_model = None + + # main llama model + self._lib = None + self._model = None + self._ctx = None + self._batch = None + self.model_path = None + self.model_params = None + self.context_params = None + + # attributes of mtmd backend lib and clip model + self._libmtmd = None + self.mtmd_ctx = None + self._clip_model = None + + self.clip_ctx = None + self.clip_model_path = "" + self.clip_base_name = "mtmd" + self._clip_base_path = "" + + self.llm_response = None + self.usage = None + self.logits = None + self.output_tokens = None + self.prompt = None + self.final_prompt = None + self._logits_all = False + + # set verbose level in environ level - will be picked up by callback in llama_cpp & mtmd + # set to "ON" to view details for debugging + os.environ["llama_cpp_verbose"] = GGUFConfigs().get_config("llama_cpp_verbose") + # e.g., os.environ["llama_cpp_verbose"] = "ON" + + self.use_sampling = sample + self.get_logits = get_logits + self.logits_record = [] + self.output_tokens = [] + self.top_logit_count = 10 + self.auto_remediate_function_call_output = True + + # default safety check in GGUF Configs that can be adjusted + gguf_configs_max = GGUFConfigs().get_config("max_output_tokens") + + if max_output > gguf_configs_max: + # truncate max output to GGUFConfigs max + logger.warning( + f"GGUFVisionGenerativeModel - requested output len - {max_output} > {gguf_configs_max}, which is the " + f"current GGUF default max.\n--Truncating to {gguf_configs_max} output tokens.\n--Note: " + f"to change GGUF default max to new integer amount, say 500:\n " + f" GGUFConfigs().set_config(\"max_output_tokens\", 500)" + ) + + max_output = gguf_configs_max + + self.max_output = max_output + self.n_seq_max = max_output + + self.target_requested_output_tokens = self.n_seq_max + self.max_total_len = 2048 + self.max_input_len = int(0.5 * context_window) + self.llm_max_output_len = int(0.5 * context_window) + self.max_output_len = self.n_seq_max + self.model_name = model_name + self.prompt_wrapper = prompt_wrapper + self.instruction_following = instruction_following + self.trailing_space = "" + self.separator = "\n" + self.eos_token_id = 0 + self.add_prompt_engineering = False + self.add_context = "" + self.model_type = "gguf" + self.model_card = model_card + self.primary_keys = None + self.function = None + self.hf_tokenizer_name = None + self.fc_supported = False + + if model_card: + + if "primary_keys" in model_card: + self.primary_keys = model_card["primary_keys"] + + if "function" in model_card: + self.function = model_card["function"] + + if "tokenizer" in model_card: + self.hf_tokenizer_name = model_card["tokenizer"] + + if "function_call" in model_card: + self.fc_supported = model_card["function_call"] + + if "trailing_space" in model_card: + self.trailing_space = model_card["trailing_space"] + else: + self.trailing_space = "" + + if "eos_token_id" in model_card: + self.eos_token_id = model_card["eos_token_id"] + + if "context_window" in model_card: + self.max_total_len = model_card["context_window"] + + if "prompt_wrapper" in model_card: + self.prompt_wrapper = model_card["prompt_wrapper"] + else: + self.prompt_wrapper = "human_bot" + + if "gguf_file" in model_card: + self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf" + + if "clip_file" in model_card: + self.clip_file = model_card["clip_file"] + + if "gguf_repo" in model_card: + self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf" + + if "instruction_following" in model_card: + self.instruction_following = model_card["instruction_following"] + + # temperature configuration + + # if temperature set at time of loading the model, then use that setting + if temperature != -99: + self.temperature = temperature + elif "temperature" in model_card: + # if not set, then pull the default temperature from the model card + self.temperature = model_card["temperature"] + else: + # if no guidance from model loading or model card, then set at GGUFConfigs default + self.temperature = GGUFConfigs().get_config("temperature_default") + + # new option to 'force' use of cuda lib, and over-ride safety checks + if GGUFConfigs().get_config("force_gpu"): + self.use_gpu = True + else: + if sys.platform.lower() not in GGUFConfigs().get_config("cuda_platforms"): + self.use_gpu = False + else: + # min drivers set to the lowest level for CUDA 12.1 on Linux + min_drivers = [525, 60] + if sys.platform.lower() == "win32": + min_drivers = GGUFConfigs().get_config("cuda_windows_driver_min") + + gpu_available = ModelCatalog().gpu_available(driver_min_levels=min_drivers) + + # use_gpu set to TRUE only if: + # (1) cuda_platform (e.g., linux or win32), e.g., not set on Mac OS + # (2) use_gpu set to True in GGUFConfigs + # (3) use_gpu_if_available flag set to True (by default) + # (4) cuda found and drivers current via direct polling of nvidia-smi executable in + # ModelCatalog.gpu_available method + + self.use_gpu = (GGUFConfigs().get_config("use_gpu") + and sys.platform.lower() in GGUFConfigs().get_config("cuda_platforms") + and gpu_available["drivers_current"] and gpu_available["gpu_found"] + and use_gpu_if_available) + + # set default minimum + self.n_batch = 2048 # alt/previous: 512 + self.last_n_tokens_size = 64 + self._n_vocab = None + self._n_ctx = None + self._token_nl = None + self._token_eos = None + self._candidates = None + self.input_ids = None + self.scores = None + self.n_tokens = 0 + self.prev = [] + self.grammar = None + + for key, value in GGUFConfigs().get_sampling_params().items(): + setattr(self, key, value) + + # no api key expected or required + self.api_key = api_key + self.api_endpoint = api_endpoint + self.error_message = "\nUnable to identify and load GGUF Vision Generative model." + self.prompt = "" + self.context = "" + self.tool_type = None + self.model_repo_path = None + self._sampler = None + self._last_image_embed = None + self._last_image_hash = None + self.file_path = "" + self.vocab = None + + # not implemented currently keeps list of tuples - (file_path, embed) + # roadmap - capture image embeddings separately for re-use + self.embed_list = [] + self.embed_tokens = [] + + self.verbose = True + + self.post_init() + + def __del__(self): + + logger.info(f"GGUFVisionGenerativeModel - cleaning up mtmd free on closing model instance") + + if self.mtmd_ctx is not None: + self._libmtmd.mtmd_free(self.mtmd_ctx) + + def load_model_for_inference(self, model_repo_path, model_card=None, **kwargs): + + """ Loads and instantiates model along with other required objects. """ + + # needs to load both llama + clip models + + if model_card: + self.model_card = model_card + + # validate before loading + self.validate() + + # load llama model + response = self._load_llama_model_for_inference(model_repo_path, model_card, **kwargs) + if not response: + logger.warning(f"GGUFVisionGenerativeModel - error loading llama backend model.") + # further triage and debug info steps ... + pass + + # load clip model + response = self._load_clip_model_for_inference(model_repo_path, model_card, **kwargs) + if not response: + logger.warning(f"GGUFVisionGenerativeModel - error loading mtmd clip backend model.") + # further triage and debug info steps ... + pass + + return self + + def _load_clip_model_for_inference(self, model_repo_path, model_card=None, **kwargs): + + """ Loads backend MTMD module along with instantiating CLIP Model and prepares associated context """ + + # load shared library + self._libmtmd = self.load_mtmd_shared_library() + self._libmtmd = add_libmtmd_ctypes_declarations(self._libmtmd) + + # set up log (best effort) - catch and skip if any errors thrown + + try: + self._libmtmd.mtmd_helper_log_set(mtmd_log_callback, ctypes.c_void_p(0)) + except: + logger.info(f"GGUFVisionGenerativeModel - unable to set mtmd log") + + ctx_params = self._libmtmd.mtmd_context_params_default() + ctx_params.use_gpu = True + ctx_params.print_timings = 0 # self.verbose + + import multiprocessing + ctx_params.n_threads = max(multiprocessing.cpu_count() // 2, 1) + + # deprecated + # ctx_params.verbosity = 0 # 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2 + + if not self.clip_file: + self.clip_file = "mmproj-F16.gguf" + + self.clip_model_path = os.path.join(model_repo_path, self.clip_file) + + # Initialize mtmd context + + self.mtmd_ctx = self._libmtmd.mtmd_init_from_file(self.clip_model_path.encode(), + self._model.model, ctx_params) + + if self.mtmd_ctx is None: + raise ValueError(f"Failed to load mtmd context from: {self.clip_model_path}") + + # Check if vision is supported + if self._libmtmd.mtmd_support_vision(self.mtmd_ctx): + + logger.info(f"GGUFVisionGenerativeModel - confirmed that model supports vision") + else: + logger.info(f"GGUFVisionGenerativeModel - model does not support vision - expect errors likely") + + return True + + def _load_llama_model_for_inference(self, model_repo_path, model_card=None, **kwargs): + + """ Loads Llama model and sets context parameters """ + + # load shared library + self._lib = self._load_llama_cpp_shared_library() + self._lib = add_ctypes_declarations(self._lib) + + if not GGUFConfigs().get_config("backend_initialized"): + # is this backend init required? + self._lib.llama_backend_init() + GGUFConfigs().set_config("backend_initialized", True) + + self._lib.llama_log_set(llama_log_callback, ctypes.c_void_p(0)) + + self.model_params = self._lib.llama_model_default_params() + + # update model params parameters + self.model_params.n_gpu_layers = 50 + + self.model_params.main_gpu = 0 + self.model_params.vocab_only = False + self.model_params.use_mmap = True + self.model_params.use_mlock = False + + if self.use_gpu: + # on darwin, keep at 0 - on win32 and linux - set to 50 by default (e.g., shift all model layers to GPU) + if sys.platform.lower() == "win32" or sys.platform.lower().startswith("linux"): + self.model_params.n_gpu_layers = GGUFConfigs().get_config("n_gpu_layers") + + # update context parameters + self.context_params = self._lib.llama_context_default_params() + + # sets minimum of 2048, but will extend if context_window is larger (e.g., 4096/8192+) + # self.context_params.n_ctx = max(2048, self.max_total_len) + self.context_params.n_ctx = 8192 # 2048 + self.context_params.n_batch = self.n_batch + + n_ubatch = 512 + self.context_params.n_ubatch = min(self.n_batch, n_ubatch) + + # note: handcrafting of thread allocation can sometimes help performance substantially + import multiprocessing + self.context_params.n_threads = max(multiprocessing.cpu_count() // 2, 1) + self.context_params.n_threads_batch = multiprocessing.cpu_count() + + # self.context_params.rope_scaling_type = (LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) + # self.context_params.pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED + self.context_params.rope_freq_base = 0.0 # (rope_freq_base if rope_freq_base != 0.0 else 0) + self.context_params.type_k = 1 + self.context_params.type_v = 1 + self.context_params.offload_kqv = True + self.context_params.yarn_orig_ctx = 0 + + if model_card: + self.model_name = model_card["model_name"].split("/")[-1] + self.gguf_file = model_card["gguf_file"] # e.g., "ggml-model-q4_k_m.gguf", + self.gguf_repo = model_card["gguf_repo"] # e.g., "llmware/dragon-mistral-7b-v0-gguf" + + self.model_path = os.path.join(model_repo_path, self.gguf_file) + + # loads and instantiates the key objects + self._model = _LlamaModel(self._lib, path_model=self.model_path, params=self.model_params) + self._ctx = _LlamaContext(self._lib, model=self._model, params=self.context_params) + self._batch = _LlamaBatch(self._lib, n_tokens=self.n_batch, embd=0, n_seq_max=self.context_params.n_ctx) + + self.vocab = self._lib.llama_model_get_vocab(self._model.model) + self._n_vocab = self.n_vocab() + self._n_ctx = self.n_ctx() + self._token_nl = self.token_nl() + self._token_eos = self.token_eos() + self._candidates = _LlamaTokenDataArray(n_vocab=self._n_vocab) + self.input_ids = np.ndarray((self._n_ctx,), dtype=np.intc) + self.scores = np.ndarray((self._n_ctx, self._n_vocab), dtype=np.single) + + self._sampler = self._init_sampler() + + return True + + def _load_llama_cpp_shared_library(self): + + """ Loads llama_cpp shared library - checks if a custom lib path has been configured - otherwise, + it loads the llmware provided dynamic libraries based on the platform/system. """ + + # check first if custom_lib_path - expected to be full path to custom so/dylib file + custom_path = GGUFConfigs().get_config("custom_lib_path") + cdll_args = dict() + + # add option to fall_back if CUDA driver can not be loaded correctly to CPU driver for that OS + fall_back_option = "" + + if custom_path: + + # point to custom llama.cpp backend libs + + if os.path.exists(custom_path): + _lib_paths = [custom_path] + else: + raise LLMWareException(message="ModuleNotFound error: could not find location of custom lib") + + else: + + _base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf") + + _lib_paths = [] + + system_platform = sys.platform.lower() + + # Determine the file extension based on the platform + if system_platform.startswith("linux"): + + # three linux versions supported - linux_x86 and linux_cuda + + machine = os.uname().machine.lower() + + if machine == "aarch64" and self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_aarch64_cuda_lib"), + GGUFConfigs().get_config("linux_cuda"))) + + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"), + GGUFConfigs().get_config("linux_cuda"))) + + # will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("linux_x86")) + + else: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("linux_x86"))) + + elif system_platform == "darwin": + + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"), + GGUFConfigs().get_config("mac_metal"))) + + elif sys.platform == "win32": + + import platform + if platform.machine().lower() == "arm64": + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"), + GGUFConfigs().get_config("windows_arm64"))) + + # windows cuda + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"), + GGUFConfigs().get_config("windows_cuda"))) + + # new - will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("windows")) + + else: + # main case - windows x86 + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("windows"))) + + else: + raise LLMWareException(message=f"No matching llama.cpp binary for platform - {system_platform}") + + # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32" and sys.version_info >= (3, 8): + os.add_dll_directory(str(_base_path)) + + # need to review + if "CUDA_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) + + cdll_args["winmode"] = ctypes.RTLD_GLOBAL + + # Try to load the shared library, handling potential errors + for _lib_path in _lib_paths: + + logger.debug(f"Loading llama cpp backend - {_lib_path}") + + if not os.path.exists(_lib_path): + if fall_back_option: + _lib_path = fall_back_option + + if os.path.exists(_lib_path): + + try: + return ctypes.cdll.LoadLibrary(str(_lib_path)) + except Exception as e: + + # if fail, and CUDA selected, then try to fall back to matching CPU version + if fall_back_option: + try: + + logger.warning("Not successful loading preferred lib so reverting to fallback lib.") + + return ctypes.cdll.LoadLibrary(str(_lib_path)) + except: + + # if fall-back fails + raise GGUFLibNotLoadedException("llama_cpp_backend", + sys.platform.lower(), + self.use_gpu, + _lib_path, + custom_path) + else: + raise GGUFLibNotLoadedException("llama_cpp_backend" ,sys.platform.lower(), + self.use_gpu, _lib_path, custom_path) + + # if not loaded + raise LLMWareException(message=f"GGUFGenerativeModel - attempting to load llama cpp backend lib - " + f"Llama cpp backend not found.") + + def _init_mtmd_context(self, llama_model): + + """Initialize mtmd context with the llama model.""" + + self.mtmd_ctx = None + + # Get default parameters + ctx_params = self._libmtmd.mtmd_context_params_default() + ctx_params.use_gpu = True # todo: expose as configuration option directly + ctx_params.print_timings = self.verbose + ctx_params.n_threads = max(multiprocessing.cpu_count() // 2, 1) + + # deprecated/removing + # ctx_params.verbosity = 2 if self.verbose else 0 # GGML_LOG_LEVEL_INFO = 2 + + # Initialize mtmd context + self.mtmd_ctx = self._libmtmd.mtmd_init_from_file(self.clip_model_path.encode(), + llama_model.model, + ctx_params) + + if self.mtmd_ctx is None: + raise ValueError(f"Failed to load mtmd context from: {self.clip_model_path}") + + # Check if vision is supported + if not self._libmtmd.mtmd_support_vision(self.mtmd_ctx): + raise ValueError("Vision is not supported by this model") + + return True + + def mtmd_free(self): + + """ Deletes MTMD context """ + + if self.mtmd_ctx is not None: + self._mtmd_cpp.mtmd_free(self.mtmd_ctx) + self.mtmd_ctx = None + + return True + + def load_mtmd_shared_library(self): + + """Platform independent shared library loader for mtmd lib backend """ + + # providing several backends packaged within llmware for the following: + + # "windows_mtmd": "mtmd.dll", + # "mac_metal_mtmd": "libmtmd.dylib", + # "linux_x86_mtmd": "libmtmd.so", + # "linux_cuda_mtmd": "libmtmd.so", + # "windows_arm64_mtmd": "mtmd.dll", + # "windows_cuda_mtmd": "mtmd.dll", + + # check first if custom_lib_path - expected to be full path to custom so/dylib file + custom_path = GGUFConfigs().get_config("custom_lib_path") + cdll_args = dict() + + fall_back_option = "" + + if custom_path: + + if os.path.exists(custom_path): + _lib_paths = [custom_path] + else: + raise LLMWareException(message="ModuleNotFound error: could not find location of custom lib") + + else: + + _base_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), "gguf") + + _lib_paths = [] + + system_platform = sys.platform.lower() + + # Determine the file extension based on the platform + if system_platform.startswith("linux"): + + # three linux versions supported - linux_x86 and linux_cuda + + machine = os.uname().machine.lower() + + if machine == "aarch64" and self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_aarch64_cuda_lib"), + GGUFConfigs().get_config("linux_cuda_mtmd"))) + + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_cuda_lib"), + GGUFConfigs().get_config("linux_cuda_mtmd"))) + + # will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("linux_x86_mtmd")) + + else: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("linux_x86_lib"), + GGUFConfigs().get_config("linux_x86_mtmd"))) + + elif system_platform == "darwin": + + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("mac_metal_lib"), + GGUFConfigs().get_config("mac_metal_mtmd"))) + + elif sys.platform == "win32": + + import platform + if platform.machine().lower() == "arm64": + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_arm64_lib"), + GGUFConfigs().get_config("windows_arm64_mtmd"))) + + # windows cuda + elif self.use_gpu: + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_cuda_lib"), + GGUFConfigs().get_config("windows_cuda_mtmd"))) + + # new - will try to use x86 as fallback + fall_back_option = os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("windows_mtmd")) + + else: + # main case - windows x86 + _lib_paths.append(os.path.join(_base_path, GGUFConfigs().get_config("windows_x86_lib"), + GGUFConfigs().get_config("windows_mtmd"))) + + else: + raise LLMWareException(message=f"No matching mtmd binary for platform - {system_platform}") + + # Add the library directory to the DLL search path on Windows (if needed) + if sys.platform == "win32" and sys.version_info >= (3, 8): + os.add_dll_directory(str(_base_path)) + + # need to review + if "CUDA_PATH" in os.environ: + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "bin")) + os.add_dll_directory(os.path.join(os.environ["CUDA_PATH"], "lib")) + + cdll_args["winmode"] = ctypes.RTLD_GLOBAL + + # Try to load the shared library, handling potential errors + for _lib_path in _lib_paths: + + logger.debug(f"Loading mtmd backend - {_lib_path}") + + if not os.path.exists(_lib_path): + if fall_back_option: + _lib_path = fall_back_option + + if os.path.exists(_lib_path): + + try: + return ctypes.cdll.LoadLibrary(str(_lib_path)) + + except Exception as e: + + # if fail, and CUDA selected, then try to fall back to matching CPU version + if fall_back_option: + + try: + + logger.warning("Not successful loading preferred lib so reverting to fallback lib.") + + return ctypes.cdll.LoadLibrary(str(_lib_path)) + + except: + + # if fall-back fails + raise GGUFLibNotLoadedException("mtmd_backend", + sys.platform.lower(), + self.use_gpu, + _lib_path, + custom_path) + else: + raise GGUFLibNotLoadedException("mtmd_backend", sys.platform.lower(), + self.use_gpu, _lib_path, custom_path) + + # if not loaded + raise LLMWareException(message=f"GGUFVisionGenerativeModel - attempting to load mtmd backend lib - " + f"mtmd backend not found.") + + + def image_to_base64_data_uri(self, file_path): + + """ Image handling utility """ + + import base64 + + with open(file_path, "rb") as img_file: + base64_data = base64.b64encode(img_file.read()).decode('utf-8') + return f"data:image/jpg;base64,{base64_data}" + + def _create_bitmap_from_bytes(self, image_bytes: bytes): + + """Create mtmd_bitmap from image bytes.""" + + if self.mtmd_ctx is None: + raise ValueError("mtmd context not initialized") + + bitmap = self._libmtmd.mtmd_helper_bitmap_init_from_buf( + self.mtmd_ctx, + (ctypes.c_uint8 * len(image_bytes)).from_buffer(bytearray(image_bytes)), + len(image_bytes) + ) + + if bitmap is None: + raise ValueError("Failed to create bitmap from image bytes") + + return bitmap + + def prepare_image_prompt(self, prompt, image_path): + + """ Main entry point for building image encodings and merging with token encodings to prepare + prompt for generative decoder model """ + + data_uri = self.image_to_base64_data_uri(image_path) + import base64 + image_bytes = base64.b64decode(data_uri.split(",")[1]) + + bitmap = self._create_bitmap_from_bytes(image_bytes) + + bitmaps = [] + bitmap_cleanup = [] + bitmaps.append(bitmap) + bitmap_cleanup.append(bitmap) + + # Create input text structure + input_text = mtmd_input_text() + input_text.text = prompt.encode('utf-8') + input_text.add_special = True + input_text.parse_special = True + + # Create input chunks + chunks = self._libmtmd.mtmd_input_chunks_init() + if chunks is None: + raise ValueError("Failed to create input chunks") + + bitmap_array = (mtmd_bitmap_p_ctypes * len(bitmaps))(*bitmaps) + + result = self._libmtmd.mtmd_tokenize( + self.mtmd_ctx, + chunks, + ctypes.byref(input_text), + bitmap_array, + len(bitmaps) + ) + + if result != 0: + raise ValueError(f"Failed to tokenize input: error code {result}") + + # Reset llama context + self.reset() + memory = self._lib.llama_get_memory(self._ctx.ctx) + self._lib.llama_memory_clear(memory, True) + + # Process each chunk + n_past = llama_pos(0) + + n_chunks = self._libmtmd.mtmd_input_chunks_size(chunks) + + for i in range(n_chunks): + + chunk = self._libmtmd.mtmd_input_chunks_get(chunks, i) + if chunk is None: + continue + + chunk_type = self._libmtmd.mtmd_input_chunk_get_type(chunk) + + if chunk_type == MTMD_INPUT_CHUNK_TYPE_TEXT: + # Handle text chunk + + n_tokens_out = ctypes.c_size_t() + tokens_ptr = self._libmtmd.mtmd_input_chunk_get_tokens_text( + chunk, ctypes.byref(n_tokens_out) + ) + + if tokens_ptr and n_tokens_out.value > 0: + # Convert ctypes array to Python list + tokens = [tokens_ptr[j] for j in range(n_tokens_out.value)] + + if self.n_tokens + len(tokens) > self.n_ctx(): + raise ValueError( + f"Prompt is larger than n_ctx: {self.n_tokens + len(tokens)} > {self.n_ctx()}" + ) + + self.eval(tokens) + + elif chunk_type in [MTMD_INPUT_CHUNK_TYPE_IMAGE, + MTMD_INPUT_CHUNK_TYPE_AUDIO]: + + chunk_n_tokens = self._libmtmd.mtmd_input_chunk_get_n_tokens(chunk) + + if self.n_tokens + chunk_n_tokens > self.n_ctx(): + raise ValueError( + f"Prompt is larger than n_ctx: {self.n_tokens + chunk_n_tokens} > {self.n_ctx()}" + ) + + new_n_past = llama_pos(0) + + result = self._libmtmd.mtmd_helper_eval_chunk_single( + self.mtmd_ctx, + self._ctx.ctx, + chunk, + llama_pos(self.n_tokens), + llama_seq_id(0), + self.n_batch, + False, # logits_last + ctypes.byref(new_n_past) + ) + + if result != 0: + raise ValueError(f"Failed to evaluate chunk: error code {result}") + + self.n_tokens = new_n_past.value + + prompt = self.input_ids[: self.n_tokens].tolist() + + self._libmtmd.mtmd_input_chunks_free(chunks) + + state_size = self._lib.llama_state_get_size(self._ctx.ctx) + + return prompt + + def _init_sampler(self): + + """ Initializes and sets up the llama cpp backend sampler """ + + # create sampler + # default params are struct + params = llama_sampler_chain_params() + self._sampler = self._lib.llama_sampler_chain_init(params) + + temp = 0.0 + + # todo: expose more sampling options + + if temp < 0.0: + # sampler.add_softmax() + self._lib.llama_sampler_chain_add(self._sampler, self._lib.llama_sampler_init_softmax()) + # sampler.add_dist(self._seed) + + elif temp == 0.0: + # sampler.add_greedy() + greedy_sampler = self._lib.llama_sampler_init_greedy() + + self._lib.llama_sampler_chain_add(self._sampler, greedy_sampler) + + return self._sampler + + def _inference(self, prompt): + + """ Tokenizes the prompt and executes generation loop. """ + + # self._sampler = self._init_sampler() + + t0 = time.time() + + completion_tokens = [] if len(prompt) > 0 else [self.token_bos()] + + prompt_tokens = ( + ( + self.tokenize(prompt.encode("utf-8"), special=True) + if prompt != "" + else [self.token_bos()] + ) + if isinstance(prompt, str) + else prompt + ) + + # confirm that input is smaller than context_window + input_len = len(prompt_tokens) + context_window = self.n_ctx() + + if input_len > context_window: + logger.warning("GGUFCLIPGenerativeModel - input is too long for model context window - " + "truncating") + min_output_len = 10 + prompt_tokens = prompt_tokens[0:context_window - min_output_len] + input_len = len(prompt_tokens) + + text = b"" + + # first token capture starts here + get_first_token_speed = GGUFConfigs().get_config("get_first_token_speed") + + token_counter = 0 + t_gen_start = time.time() + first_token_processing_time = -1.0 + + for token in self.generate(prompt_tokens): + + # first token capture + if get_first_token_speed: + if token_counter == 0: + first_token_processing_time = time.time() - t_gen_start + token_counter += 1 + # first token capture ends here + + if self.get_logits: + self.register_top_logits() + self.output_tokens.append(token) + + if token == self._token_eos: + text = self.detokenize(completion_tokens) + break + + completion_tokens.append(token) + + # stop at max output len + if len(completion_tokens) >= self.max_output_len: + text = self.detokenize(completion_tokens) + break + + # stop if combined input + output at context window size + if (input_len + len(completion_tokens)) >= context_window: + text = self.detokenize(completion_tokens) + break + + text_str = text.decode("utf-8", errors="ignore") + + # post-processing clean-up - stop at endoftext + eot = text_str.find("<|endoftext|>") + if eot > -1: + text_str = text_str[:eot] + + # new post-processing clean-up - stop at + eots = text_str.find("") + if eots > -1: + text_str = text_str[:eots] + + # post-processing clean-up - start after bot wrapper + bot = text_str.find(":") + if bot > -1: + text_str = text_str[bot + len(":"):] + + # new post-processing cleanup - skip repeating starting + boss = text_str.find("") + if boss > -1: + text_str = text_str[boss + len(""):] + + # end - post-processing + + if get_first_token_speed: + + output = {"llm_response": text_str, + "usage": {"input": len(prompt_tokens), "output": len(completion_tokens), + "total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens", + "processing_time": time.time() - t0, + "first_token_processing_time": first_token_processing_time}} + else: + output = {"llm_response": text_str, + "usage": {"input": len(prompt_tokens), "output": len(completion_tokens), + "total": len(prompt_tokens) + len(completion_tokens), "metric": "tokens", + "processing_time": time.time() - t0}} + + if self.get_logits: + output.update({"logits": self.logits_record, "output_tokens": self.output_tokens}) + + return output + + def sample_gguf(self, idx=None): + + """ Adapted to sample_gguf to avoid potential name space conflicts. """ + + # assert self.n_tokens > 0 + + tmp_sampler = False + + if self._sampler is None: + tmp_sampler = True + self._sampler = self._init_sampler() + + ridx = idx - self.n_tokens if idx is not None else -1 + + assert self.ctx is not None + + token = self._lib.llama_sampler_sample(self._sampler, self._ctx.ctx, ridx) + + # token = int(self.logits_record[-1][0][0]) + + if tmp_sampler: + self._sampler = None + + return token + + def generate(self, tokens, reset=True): + + """ Generator that samples the model and yields tokens until stopped. """ + + # test + + # Check for kv cache prefix match + if reset and self.n_tokens > 0: + longest_prefix = 0 + for a, b in zip(self._input_ids, tokens[:-1]): + if a == b: + longest_prefix += 1 + else: + break + if longest_prefix > 0: + reset = False + tokens = tokens[longest_prefix:] + self.n_tokens = longest_prefix + + # Reset the model state + # reset = False + if reset: + self.reset() + + sample_idx = self.n_tokens + len(tokens) - 1 + tokens = list(tokens) + + tokens_created = 0 + input_start_len = len(tokens) + + memory = self._ctx.memory + + # Eval and sample + while True: + + self._lib.llama_memory_seq_rm(memory, -1, self.n_tokens, -1) + + for i in range(0, len(tokens), self.n_batch): + batch = tokens[i: min(len(tokens), i + self.n_batch)] + n_past = self.n_tokens + n_tokens = len(batch) + + self._batch.set_batch(batch=batch, n_past=n_past, logits_all=self._logits_all) + + return_code = self._lib.llama_decode(self._ctx.ctx, self._batch.batch) + + # TODO: add better error handling if return_code 1 - usually overflow of ctx + if return_code != 0: + raise RuntimeError(f"llama_decode call returned {return_code} - in most cases, this " + f"is due to exceeding the maximum context window.") + + self.input_ids[n_past: n_past + n_tokens] = batch + rows = n_tokens + cols = self._n_vocab + offset = (0 if self._logits_all else n_tokens - 1) + + if self._logits_all: + rows = n_tokens + cols = self._n_vocab + logits = np.ctypeslib.as_array( + self._ctx.get_logits(), shape=(rows * cols,)) + self.scores[n_past: n_past + n_tokens, :].reshape(-1)[::] = logits + + self.n_tokens += n_tokens + + # leaving hard-coded off for now (improves performance) + # self.register_top_logits() + + while sample_idx < self.n_tokens: + + logits = self._scores[-1, :] + + self.prev = list(self.eval_tokens) + + # sample to generate token from logits + token = self.sample_gguf(idx=sample_idx) # (logits_array=logits) + + self.accept(id=id, apply_grammar=None) + + tokens_created += 1 + + sample_idx += 1 + + tokens_or_none = yield token + tokens.clear() + tokens.append(token) + if tokens_or_none is not None: + tokens.extend(tokens_or_none) + + if sample_idx < self.n_tokens and token != self._input_ids[sample_idx]: + self.n_tokens = sample_idx + + self._lib.llama_memory_seq_rm(self._lib.llama_get_memory(self._ctx.ctx), -1, self.n_tokens, -1) + break + + if tokens_created > self.max_output_len: + logger.info("GGUFVisionGenerativeModel - stopping generation loop - reached limit of " + "max output len") + break + + def tokenize(self, text, add_bos=True, special=False): + + """ Tokenizes text. """ + + n_ctx = self.n_ctx_train() + tokens = (ctypes.c_int32 * n_ctx)() + # change from self._model.model + n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_ctx, add_bos, special) + + if n_tokens < 0: + n_tokens = abs(n_tokens) + tokens = (ctypes.c_int32 * n_tokens)() + + n_tokens = self._lib.llama_tokenize(self.vocab, text, len(text), tokens, n_tokens, add_bos, special) + + if n_tokens < 0: + raise RuntimeError(f"GGUFVisionGenerativeModel - tokenization error - " + f"{text} - n_tokens={n_tokens}") + + return list(tokens[:n_tokens]) + + def detokenize(self, tokens, special: bool = False) -> bytes: + output = b"" + size = 32 + buffer = (ctypes.c_char * size)() + for token in tokens: + n = self._lib.llama_token_to_piece( + # replace: self.model + self.vocab, llama_token(token), buffer, size, 0, special + ) + assert n <= size + output += bytes(buffer[:n]) + + # following llama_cpp_python on below ... + # NOTE: Llama1 models automatically added a space at the start of the prompt + # this line removes a leading space if the first token is a beginning of sentence token + + return ( + output[1:] + if len(tokens) > 0 and tokens[0] == self.token_bos() and output[0:1] == b" " + else output + ) + + def accept(self, id, apply_grammar): + + """ Formal step post sampling that 'accepts' and adds the token id to the running generation. """ + + if apply_grammar and self.grammar is not None: + self._lib.llama_grammar_accept_token(self._ctx.ctx, self.grammar.grammar, id) + + self.prev.append(id) + + def register_top_logits(self): + + """ Gets the top logits and keeps a running log for output analysis. """ + + # TODO: there is issue with first logit computation - not corresponding to first token + logit_pointer = self._lib.llama_get_logits(self._ctx.ctx) + + logit_size = self.n_vocab() + logit_array = np.zeros(logit_size) + for x in range(0, logit_size): + logit_array[x] = logit_pointer[x] + + sm = np.exp(logit_array) / sum(np.exp(logit_array)) + + sm_sorted = np.sort(sm) + sm_args_sorted = np.argsort(sm) + + top_logits = [] + + for x in range(0, self.top_logit_count): + # experiment - try rounding the float number + pair = (sm_args_sorted[logit_size - x - 1], round(sm_sorted[logit_size - x - 1], 3)) + top_logits.append(pair) + # print("--test: logits - ", x, top_logits) + + self.logits_record.append(top_logits) + + return top_logits + + def set_api_key(self, api_key, env_var="USER_MANAGED_GGUF_API_KEY"): + + """ Sets API key - generally not used in GGUF models. """ + + # set api_key + os.environ[env_var] = api_key + logger.info("added and stored GGUF api_key in environmental variable- %s", env_var) + + return self + + def _get_api_key(self, env_var="USER_MANAGED_GGUF_API_KEY"): + + """ Gets API key - generally not used in GGUF models. """ + + self.api_key = os.environ.get(env_var) + + if not self.api_key: + logger.warning("_get_api_key could not successfully retrieve value from: %s ", env_var) + + return self.api_key + + @property + def ctx(self): + return self._ctx.ctx + + @property + def model(self): + return self._model.model + + @property + def _input_ids(self): + return self.input_ids[: self.n_tokens] + + @property + def _scores(self): + return self.scores[: self.n_tokens, :] + + @property + def eval_tokens(self): + return deque(self.input_ids[: self.n_tokens].tolist(), maxlen=self._n_ctx) + + def eval(self, tokens): + + """Evaluate a list of tokens. + + Args: + tokens: The list of tokens to evaluate. + """ + + memory = self._ctx.memory + self._lib.llama_memory_seq_rm(memory, -1, self.n_tokens, -1) + + for i in range(0, len(tokens), self.n_batch): + batch = tokens[i: min(len(tokens), i + self.n_batch)] + n_past = self.n_tokens + n_tokens = len(batch) + self._batch.set_batch( + batch=batch, n_past=n_past, logits_all=self._logits_all + ) + + self._lib.llama_decode(self._ctx.ctx, self._batch.batch) + + # Save tokens + self.input_ids[n_past: n_past + n_tokens] = batch + + # Save logits + if self._logits_all: + rows = n_tokens + cols = self._n_vocab + logits = np.ctypeslib.as_array( + self._ctx.get_logits(), shape=(rows * cols,) + ) + self.scores[n_past: n_past + n_tokens, :].reshape(-1)[::] = logits + else: + pass + # Update n_tokens + + self.n_tokens += n_tokens + + @property + def eval_logits(self): + return deque( + self.scores[: self.n_tokens, :].tolist(), + maxlen=self._n_ctx if self._logits_all else 1, + ) + + def reset(self): + self.n_tokens = 0 + + def n_ctx(self): + return self._lib.llama_n_ctx(self._ctx.ctx) + + def n_ctx_train(self): + return self._lib.llama_n_ctx_train(self._model.model) + + def n_vocab(self): + # llama_model_get_vocab(model) + n_vocab = self._lib.llama_n_vocab(self._lib.llama_model_get_vocab(self._model.model)) + return n_vocab + + def token_eos(self): + # return self._lib.llama_token_eos(self._model.model) + eos = self._lib.llama_token_eos(self.vocab) + return eos + + def token_bos(self): + # return self._lib.llama_token_bos(self._model.model) + bos = self._lib.llama_token_bos(self.vocab) + return bos + + def token_nl(self): + token_nl = self._lib.llama_token_nl(self._lib.llama_model_get_vocab(self._model.model)) + # return self._lib.llama_token_nl(self._model.model) + return token_nl + + def unload_model(self): + + """ Unloads a model to release memory """ + + # note: removing pointer seems to safely remove from Python reference tracking + + self._batch = None + self._ctx = None + self._model = None + + return 0 + + def inference(self, prompt, image_path, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None, get_logits=False, disable_eos=False): + + """ Main method for inference generation. """ + + logger.info("GGUFVisionGenerativeModel - Starting generation inference") + + time_start = time.time() + + media_marker = self._libmtmd.mtmd_default_marker().decode('utf-8') + text = "\n" + str(media_marker) + prompt + + self.prompt = text + prompt = self.prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # update default handling for no add_prompt_engineering + + if not self.add_prompt_engineering: + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # start with clean logits_record and output_tokens for each function call + self.logits_record = [] + self.output_tokens = [] + + if get_logits: + self.get_logits = get_logits + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # preview before generation + # self.preview() + + # prompt = prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + prompt = prompt_final + self.trailing_space + + # prepare embedded image prompt with fully templated prompt + prompt_tokens = self.prepare_image_prompt(prompt, image_path) + + # output_response = self._inference(text_prompt) + + # starts _inference here + completion_tokens = [] if len(prompt_tokens) > 0 else [self.token_bos()] + + # todo: safety checks to confirm that input is smaller than context_window + input_len = len(prompt_tokens) + context_window = self.n_ctx() + + text = b"" + + token_list = [] + token_counter = 0 + text_output = "" + + for token in self.generate(prompt_tokens): + + completion_tokens.append(token) + + if not disable_eos: + if token == self._token_eos: + break + + if len(completion_tokens) > self.max_output_len: + break + + # stop if combined input + output at context window size + if (input_len + len(completion_tokens)) >= context_window: + break + + new_token = self.detokenize([token]).decode('utf-8', errors='ignore') + + text_output += new_token + token_counter += 1 + + # text_str = text_output.decode("utf-8", errors="ignore") + + usage = {"input": input_len, + "output": token_counter, + "total": input_len + token_counter, + "metric": "tokens", + "processing_time": time.time() - time_start} + + response = {"llm_response": text_output, "usage": usage} + + self.register() + + return response + + def stream(self, prompt, image_path, add_context=None, add_prompt_engineering=None, api_key=None, + inference_dict=None, + get_logits=False, disable_eos=False): + + """ Main method for text streaming generation. Returns a generator function that yields one + token at a time for real-time streaming to console or UI. """ + + logger.info("GGUFVisionGenerativeModel - Starting generation stream") + + media_marker = self._libmtmd.mtmd_default_marker().decode('utf-8') + text = "\n" + str(media_marker) + prompt + + self.prompt = text + prompt = self.prompt + + if add_context: + self.add_context = add_context + + if add_prompt_engineering: + self.add_prompt_engineering = add_prompt_engineering + + # update default handling for no add_prompt_engineering + + if not self.add_prompt_engineering: + if self.add_context: + self.add_prompt_engineering = "default_with_context" + else: + self.add_prompt_engineering = "default_no_context" + + # start with clean logits_record and output_tokens for each function call + self.logits_record = [] + self.output_tokens = [] + + if get_logits: + self.get_logits = get_logits + + if inference_dict: + + if "temperature" in inference_dict: + self.temperature = inference_dict["temperature"] + + if "max_tokens" in inference_dict: + self.target_requested_output_tokens = inference_dict["max_tokens"] + + # preview before generation + # self.preview() + + # prompt = prompt + + if self.add_prompt_engineering: + prompt_enriched = self.prompt_engineer(self.prompt, self.add_context, inference_dict=inference_dict) + prompt_final = prompt_enriched + + # most models perform better with no trailing space or line-break at the end of prompt + # -- in most cases, the trailing space will be "" + # -- yi model prefers a trailing "\n" + # -- keep as parameterized option to maximize generation performance + # -- can be passed either thru model_card or model config from HF + + prompt = prompt_final + self.trailing_space + + # prepare embedded image prompt with fully templated prompt + prompt_tokens = self.prepare_image_prompt(prompt, image_path) + + # output_response = self._inference(text_prompt) + + # starts _inference here + completion_tokens = [] if len(prompt_tokens) > 0 else [self.token_bos()] + + #todo: safety checks to confirm that input is smaller than context_window + input_len = len(prompt_tokens) + context_window = self.n_ctx() + + text = b"" + + # disable_eos = True + token_list = [] + + for token in self.generate(prompt_tokens): + + completion_tokens.append(token) + + if not disable_eos: + if token == self._token_eos: + break + + if len(completion_tokens) > self.max_output_len: + break + + # stop if combined input + output at context window size + if (input_len + len(completion_tokens)) >= context_window: + break + + new_token = self.detokenize([token]).decode('utf-8', errors='ignore') + + yield new_token + + text_str = text.decode("utf-8", errors="ignore") + + # turned off + self.register() + + return text_str + + + def function_call(self, context, function=None, params=None,get_logits=True,temperature=-99.0,max_output=None): + """ Not implemented for this model class. """ + return True + + def function_call_over_api_endpoint(self, context="", tool_type="", model_name="", params="", prompt="", + function=None, endpoint_base=None, api_key=None, get_logits=False): + """ Not implemented for this model class """ + return True + + def inference_over_api_endpoint(self, prompt, context=None, inference_dict=None, get_logits=False): + """ Not implemented for this model class """ + return True diff --git a/llmware/parsers.py b/llmware/parsers.py new file mode 100644 index 0000000..e5b549e --- /dev/null +++ b/llmware/parsers.py @@ -0,0 +1,4779 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + + +"""The parsers module implements all parsers, i.e. all conversions fom an unstructured document/content type into +set of text chunks, e.g., blocks, indexed with metadata, in a database. + + Parsers can be accessed through at least 3 distinct entry points: + + 1. Library 'add_files' - avoids the need to explicitly instantiate a Parser object, as the Parser is + instantiated indirectly through the Library class with its convenience universal 'add_files' to Parser + 'ingest' method that collates and parses any supported file types found in the input folder. + This is the easiest way to handle large-scale ingestion, especially with multiple file types. + + 2. Explicit Parser + Library - create a Parser object, and directly pass a Library object, and then + use specific parsing methods to parse particular document types into the Library. This is useful for + document types where you would like to control the parser parameters, such as a custom csv, custom json, + or aws transcript. It can also be useful in special situations to have more explicit control of a + specific parsing parameter. + + 3. Parse to File - create a Parser object without a Library, and the parsing will be saved in the ParserState, + and available as a consolidated JSON file. You can also retrieve parsing outputs in memory, as a list of + dictionaries, that can be handled directly without any storage. + + The module currently implements parsers for PDF, Office (DOCX, PPTX, XLSX), CSV, JSON/JSONL, MD, TSV, + WAV, PNG, JPEG, HTML WebSites, and AWS voice transcripts. +""" + +import time +import json +import os +from zipfile import ZipFile, ZIP_DEFLATED +import shutil + +import logging +import random +from ctypes import * +import platform + +from llmware.configs import LLMWareConfig, LLMWareTableSchema, LLMWareException, DependencyNotInstalledException +from llmware.util import Utilities, TextChunker +from llmware.web_services import WikiKnowledgeBase, WebSiteParser +from llmware.resources import CollectionRetrieval, CollectionWriter, ParserState + +logger = logging.getLogger(__name__) +logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__)) + + +class Parser: + + def __init__(self, library=None, account_name="llmware", parse_to_db=False, file_counter=1, + encoding="utf-8", chunk_size=400, max_chunk_size=600, smart_chunking=1, + get_images=True, get_tables=True, strip_header=False, table_grid=True, + get_header_text=True, table_strategy=1, verbose_level=2, copy_files_to_library=True, + set_custom_logging=-1, use_logging_file=False): + + """ Main class for handling parsing, e.g., conversion of documents and other unstructured files + into indexed text collection of 'blocks' in database. For most use cases, Parser does not need + to be invoked directly - as Library and Prompt are more natural client interfaces. """ + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + # new path for parser history - records of parse job outputs (outside of library construct) + self.parser_folder = LLMWareConfig.get_parser_path() + + if not os.path.exists(self.parser_folder): + os.mkdir(self.parser_folder) + + # create tmp workspace for parser + tmp_path = LLMWareConfig.get_tmp_path() + parser_tmp_work_folder = os.path.join(tmp_path, "parser_tmp" + os.sep) + + # if tmp path not in place, explicitly create + if not os.path.exists(tmp_path): + os.mkdir(tmp_path) + os.chmod(tmp_path, 0o777) + + # if tmp workspace folder already exists, then delete - start fresh + if os.path.exists(parser_tmp_work_folder): + shutil.rmtree(parser_tmp_work_folder) + os.mkdir(parser_tmp_work_folder) + + self.parser_tmp_folder = parser_tmp_work_folder + + # shift library to optional parameter - allows calls to Parser class without a library declared + self.account_name = account_name + + # placeholder used if no library passed in constructor + self.library_name = "default" + + self.library = library + self.block_size_target_characters = 600 + + # will track and increment files processed within same parsing job + self.file_counter = file_counter + + # by default, parse_to_db = False + self.parse_to_db = parse_to_db + + self.parser_job_id = ParserState().issue_new_parse_job_id() + + # if library is passed to parser, then assumes will write to library db, if available + if library: + self.account_name = library.account_name + self.library_name = library.library_name + self.block_size_target_characters = library.block_size_target_characters + + self.parser_image_folder = library.image_path + + # sets parse_to_db == True only if (a) library passed in constructor, and (b) collection db found + + # check if collection datastore is connected + if CollectionRetrieval(self.library_name,account_name=self.account_name).test_connection(): + # if not check_db_uri(timeout_secs=3): + self.parse_to_db = True + else: + logger.warning(f"warning: Parser not able to connect to document store collection database " + f"at uri - {LLMWareConfig.get_db_uri_string()} - will write parsing output to " + f"a parsing file.") + + self.parse_to_db = False + else: + # if no library passed + self.parse_to_db = False + self.parser_image_folder = self.parser_tmp_folder + + # used to pass to the C parsers in pdf/office parsing paths + self.collection_path = LLMWareConfig.get_db_uri_string() + self.collection_db_configs = LLMWareConfig.get_db_configs() + self.collection_db_username = LLMWareConfig.get_db_user_name() + self.collection_db_password = LLMWareConfig.get_db_pw() + + # 'active' output state tracker + self.parser_output = [] + + self.ACCEPTED_FILE_FORMATS = ["pptx","xlsx","docx","pdf","txt","csv","html","jsonl", + "jpg","jpeg","png","wav","zip", "md", "tsv"] + self.office_types = ["PPTX", "pptx", "XLSX", "xlsx", "DOCX", "docx"] + self.pdf_types = ["PDF", "pdf"] + self.text_types = ["txt", "csv", "html", "jsonl", "md", "tsv"] + self.ocr_types = ["jpg", "jpeg", "png"] + self.voice_types = ["wav", "mp3", "mp4", "m4a"] + self.zip_types = ["zip"] + self.office_work_folder = None + self.pdf_work_folder = None + self.text_work_folder = None + self.voice_work_folder = None + self.zip_work_folder = None + self.ocr_work_folder = None + self.dialog_work_folder = None + self.website_work_folder = None + self.supported_parser_types = ["pdf", "office", "text", "voice", "dialog", "web", "image", + "pdf_by_ocr"] + + self.schema = LLMWareTableSchema.get_parser_table_schema() + + if self.parse_to_db: + # if table does not exist, then create + if CollectionWriter("parser_events", account_name=self.account_name).check_if_table_build_required(): + + # create "status" table + CollectionWriter("parser_events", account_name=self.account_name).create_table("parser_events", + self.schema) + + # parsers write status update - confirm that status tables created + if CollectionWriter("status", account_name=self.account_name).check_if_table_build_required(): + + CollectionWriter("status", + account_name=self.account_name).create_table("status", + LLMWareTableSchema.get_status_schema()) + + # new parameters + self.encoding=encoding + self.chunk_size = chunk_size + self.max_chunk_size = max_chunk_size + self.smart_chunking = smart_chunking + self.get_images = get_images + self.get_tables = get_tables + self.strip_header = strip_header + self.table_grid = table_grid + self.table_strategy= table_strategy + self.get_header_text = get_header_text + self.verbose_level = verbose_level + self.copy_files_to_library = copy_files_to_library + + # new logging + if set_custom_logging > -1: + self.logger_level = set_custom_logging + logger.info(f"Parser constructor - setting custom logging level - {self.logger_level}") + else: + self.logger_level = LLMWareConfig().get_logging_level_by_module(__name__) + + self.parser_log_name = "parser_log.txt" + self.use_logging_file = use_logging_file + + def clear_state(self): + + """Clears parser state. """ + + self.parser_output = [] + return self + + def save_state(self): + + """ Saves parser state. """ + + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + return self + + def _setup_workspace(self, local_work_path): + + """ Internal method to setup workspace for parsing job. """ + + # set up local workspace folders + if not local_work_path: + + if self.library: + local_work_path = self.library.tmp_path + else: + # if no library selected, then default to parser_tmp_folder + local_work_path = self.parser_tmp_folder + + if not os.path.exists(local_work_path): + os.makedirs(local_work_path, exist_ok=True) + + office_fp = os.path.join(local_work_path, "process_office_files" + os.sep) + pdf_fp = os.path.join(local_work_path, "process_pdf_files" + os.sep) + text_fp = os.path.join(local_work_path, "process_text_files" + os.sep) + ocr_fp = os.path.join(local_work_path, "process_ocr_files" + os.sep) + voice_fp = os.path.join(local_work_path, "process_voice_files" + os.sep) + zip_fp = os.path.join(local_work_path, "process_zip_files" + os.sep) + + office_workspace_fp = os.path.join(local_work_path, "office_tmp" + os.sep) + + # start clean with new directories for both office + pdf + if os.path.exists(office_fp): + shutil.rmtree(office_fp, ignore_errors=True) + os.mkdir(office_fp) + self.office_work_folder = office_fp + + if os.path.exists(pdf_fp): + shutil.rmtree(pdf_fp, ignore_errors=True) + os.mkdir(pdf_fp) + self.pdf_work_folder = pdf_fp + + if os.path.exists(text_fp): + shutil.rmtree(text_fp, ignore_errors=True) + os.mkdir(text_fp) + self.text_work_folder = text_fp + + if os.path.exists(ocr_fp): + shutil.rmtree(ocr_fp, ignore_errors=True) + os.mkdir(ocr_fp) + self.ocr_work_folder = ocr_fp + + if os.path.exists(voice_fp): + shutil.rmtree(voice_fp, ignore_errors=True) + os.mkdir(voice_fp) + self.voice_work_folder = voice_fp + + if os.path.exists(zip_fp): + shutil.rmtree(zip_fp, ignore_errors=True) + os.mkdir(zip_fp) + self.zip_work_folder = zip_fp + + if os.path.exists(office_workspace_fp): + shutil.rmtree(office_workspace_fp, ignore_errors=True) + os.mkdir(office_workspace_fp) + self.office_tmp = office_workspace_fp + + def _collator(self, input_folder_path, dupe_check=False): + + """ Internal utility method to prepare and organize files for parsing. """ + + # run comparison for existing files if dupe_check set True + # default case - no checking for dupes + existing_files = [] + + # run comparison for existing files if dupe_check set True + if self.library: + if dupe_check and os.path.exists(self.library.file_copy_path): + existing_files = os.listdir(self.library.file_copy_path) + + # counters + dup_counter = 0 + office_found = 0 + pdf_found = 0 + zip_found = 0 + text_found = 0 + ocr_found = 0 + voice_found = 0 + + # list of input files + input_file_names = os.listdir(input_folder_path) + files_to_be_processed = [] + duplicate_files = [] + + if dupe_check: + # we get a reduced list of input_file_names if in existing_files is files we try to process + duplicate_files_tmp = list(set(input_file_names) - set(existing_files)) + # the duplicates are those that where not in duplicate_files_tmp so we take out the tmp from the input_file_names + # what's left is the duplicates + duplicate_files = list(set(input_file_names) - set(duplicate_files_tmp)) + # the counter is the length of the array + dup_counter = len(duplicate_files) + # We are done with this and we don't need to n times loop as before + # we set the imput_file_names to be the reduced list to not to process dupe files + input_file_names = duplicate_files_tmp + + for filename in input_file_names: + + filetype = filename.split(".")[-1] + + files_to_be_processed.append(filename) + + # copy file into specific channel for targeted parser + + if filetype.lower() in self.office_types: + shutil.copy(os.path.join(input_folder_path,filename), os.path.join(self.office_work_folder,filename)) + office_found += 1 + + if filetype.lower() in self.pdf_types: + shutil.copy(os.path.join(input_folder_path,filename), os.path.join(self.pdf_work_folder, filename)) + pdf_found += 1 + + if filetype.lower() in self.text_types: + shutil.copy(os.path.join(input_folder_path,filename), os.path.join(self.text_work_folder,filename)) + text_found += 1 + + if filetype.lower() in self.ocr_types: + shutil.copy(os.path.join(input_folder_path,filename), os.path.join(self.ocr_work_folder,filename)) + ocr_found += 1 + + if filetype.lower() in self.voice_types: + shutil.copy(os.path.join(input_folder_path,filename), os.path.join(self.voice_work_folder,filename)) + voice_found += 1 + + if filetype.lower() in self.zip_types: + shutil.copy(os.path.join(input_folder_path,filename), os.path.join(self.zip_work_folder,filename)) + zip_found += 1 + + logger.info(f"update: Duplicate files (skipped): {dup_counter}") + logger.info(f"update: Total uploaded: {len(input_file_names)}") + + if zip_found > 0: + + # if any zip files found in upload, then unpack and process first + # --once zip extracted, push all files into the appropriate work folder for pdf, office, etc. + # --inside zip_extract_handler- will update counters + + zip_work_order = self.zip_extract_handler() + pdf_found += zip_work_order["pdf"] + office_found += zip_work_order["office"] + text_found += zip_work_order["text"] + voice_found += zip_work_order["voice"] + ocr_found += zip_work_order["ocr"] + + work_order = {"pdf": pdf_found, + "office": office_found, + "text": text_found, + "ocr": ocr_found, + "voice": voice_found, + "duplicate_files": duplicate_files, + "file_list": files_to_be_processed} + + return work_order + + def ingest (self, input_folder_path, dupe_check=True): + + """ Main method for large-scale parsing. Takes only a single input which is the local input folder path + containing the files to be parsed. + + Optional dupe_check parameter set to True to restrict ingesting a file with the same name as a file + already in the library. """ + + # input_folder_path = where the input files are located + + # first - confirm that library and connection to collection db are in place + if not self.library or not self.parse_to_db: + + logger.error("error: Parser().ingest() method requires loading a library, e.g., " + "Parser(library=my_library), and a connection to a document data store - please " + "try Parse().parse_one set of methods to parse a document of any type directly into " + "list of dictionaries in memory, and written to /parser_history as a .json file") + + parsing_results = {"processed_files": 0, "rejected_files": 0, "duplicate_files": []} + return parsing_results + + # prepares workspace for individual parsers + self._setup_workspace(self.parser_tmp_folder) + + # collate and sort the file types in the work path + work_order = self._collator(input_folder_path, dupe_check=dupe_check) + + # write to db - True only if library loaded + collection connect in place + write_to_db = self.parse_to_db + + if work_order["office"] > 0: + self.parse_office(self.office_work_folder, save_history=False) + + if self.copy_files_to_library: + self.uploads(self.office_work_folder) + + if work_order["pdf"] > 0: + self.parse_pdf(self.pdf_work_folder, save_history=False) + + if self.copy_files_to_library: + self.uploads(self.pdf_work_folder) + + if work_order["text"] > 0: + self.parse_text(self.text_work_folder, save_history=False) + + if self.copy_files_to_library: + self.uploads(self.text_work_folder) + + if work_order["ocr"] > 0: + self.parse_image(self.ocr_work_folder, save_history=False) + + if self.copy_files_to_library: + self.uploads(self.ocr_work_folder) + + if work_order["voice"] > 0: + self.parse_voice(self.voice_work_folder, save_history=False) + + if self.copy_files_to_library: + self.uploads(self.voice_work_folder) + + # need to systematically capture list of rejected docs + + processed, not_processed = self.input_ingestion_comparison(work_order["file_list"]) + + parsing_results = {"processed_files": processed, + "rejected_files": not_processed, + "duplicate_files": work_order["duplicate_files"]} + + return parsing_results + + def ingest_to_json(self, input_folder_path): + + """ Mirrors the main ingest method but intended for writing parsing output directly to json when + 'writing_to_db' = False. """ + + # prepares workspace for individual parsers + self._setup_workspace(self.parser_tmp_folder) + + # collate and sort the file types in the work path + work_order = self._collator(input_folder_path, dupe_check=False) + + # write to db - True only if library loaded + collection connect in place + self.parse_to_db = False + self.library = None + + if work_order["office"] > 0: + self.parse_office(self.office_work_folder, write_to_db=False, save_history=False) + + if work_order["pdf"] > 0: + self.parse_pdf(self.pdf_work_folder, write_to_db=False, save_history=False) + + if work_order["text"] > 0: + self.parse_text(self.text_work_folder, write_to_db=False, save_history=False) + + if work_order["ocr"] > 0: + self.parse_image(self.ocr_work_folder, write_to_db=False, save_history=False) + + if work_order["voice"] > 0: + self.parse_voice(self.voice_work_folder, write_to_db=False, save_history=False) + + # need to systematically capture list of rejected docs + + fn = ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + processed, not_processed = self.input_ingestion_comparison_from_parser_state(work_order["file_list"]) + + parsing_results = {"processed_files": processed, + "rejected_files": not_processed, + "parser_output_filename": fn} + + return parsing_results + + def parse_by_type(self, parser_type, input_folder_path, url=None): + + """ Parse files by content type. """ + + output = None + + if parser_type in self.supported_parser_types: + + if parser_type == "pdf": + output = self.parse_pdf(input_folder_path, write_to_db=self.parse_to_db) + + if parser_type == "office": + output = self.parse_office(input_folder_path, write_to_db=self.parse_to_db) + + if parser_type == "text": + output = self.parse_text(input_folder_path, write_to_db=self.parse_to_db) + + if parser_type == "voice": + output = self.parse_voice(input_folder_path, write_to_db=self.parse_to_db) + + if parser_type == "dialog": + output = self.parse_dialog(input_folder_path, write_to_db=self.parse_to_db) + + if parser_type == "web": + output = self.parse_website(url, write_to_db=self.parse_to_db) + + if parser_type == "pdf_by_ocr": + output = self.parse_pdf_by_ocr_images(input_folder_path, write_to_db=self.parse_to_db) + + return output + + def zip_extract_handler(self): + + """ Unzips and extracts files from zip archive -and iteratively push files to specific file path. """ + + # tracker for files found inside the zip + pdf_found = 0 + office_found = 0 + text_found = 0 + ocr_found = 0 + voice_found = 0 + + z = "" + + zip_files = os.listdir(self.zip_work_folder) + + for my_zip_names in zip_files: + + # iterate thru all of the .zip files found + + my_zip = self.zip_work_folder + my_zip_names + + # create fresh /tmp file to extract the zip files + if os.path.exists(os.path.join(self.zip_work_folder,"tmp")): + shutil.rmtree(os.path.join(self.zip_work_folder,"tmp"), ignore_errors=True) + os.mkdir(os.path.join(self.zip_work_folder,"tmp")) + + try: + # unzip and extract into /tmp folder + z = ZipFile(my_zip, 'r', compression=ZIP_DEFLATED) + ZipFile.extractall(z, os.path.join(self.zip_work_folder, "tmp")) + success_code = 1 + + except: + # may fail + success_code = -1 + logger.info(f"error: caution - could not open Zip- {my_zip}") + + if success_code == 1: + + # iterate thru all of the files found in the zip archive + # apply secure filename and prep filename + # route to the appropriate work folder, if applicable + + for f in z.namelist(): + + # will apply secure name and cap length, but does not run duplicate file check + fn = self.prep_filename(f, max_len=240, secure_name=True) + ext = fn.split(".")[-1] + + if success_code == 1: + + if ext in self.office_types: + shutil.copy(os.path.join(self.zip_work_folder,"tmp" + os.sep,f), + os.path.join(self.office_work_folder,fn)) + office_found += 1 + + if ext in self.pdf_types: + shutil.copy(os.path.join(self.zip_work_folder, "tmp" + os.sep, f), + os.path.join(self.pdf_work_folder,fn)) + pdf_found += 1 + + if ext in self.text_types: + shutil.copy(os.path.join(self.zip_work_folder, "tmp" + os.sep, f), + os.path.join(self.text_work_folder,fn)) + text_found += 1 + + if ext in self.ocr_types: + shutil.copy(os.path.join(self.zip_work_folder,"tmp" + os.sep,f), + os.path.join(self.ocr_work_folder,fn)) + ocr_found += 1 + + if ext in self.voice_types: + shutil.copy(os.path.join(self.zip_work_folder,"tmp" + os.sep,f), + os.path.join(self.voice_work_folder, fn)) + voice_found += 1 + + work_order = {"pdf": pdf_found, "office": office_found, "text": text_found, "ocr": ocr_found, "voice": voice_found} + + return work_order + + def convert_parsing_txt_file_to_json(self, file_path=None, fn="pdf_internal_test0.txt"): + + """ Utility method that picks up a .txt file output from Office or PDF parser and converts to a list + of dictionaries for insertion in an external DB. """ + + default_keys = ["block_ID", "doc_ID", "content_type", "file_type", "master_index", "master_index2", + "coords_x", "coords_y", "coords_cx", "coords_cy", "author_or_speaker", "modified_date", + "created_date", "creator_tool", "added_to_collection", "file_source", + "table", "external_files", "text", "header_text", "text_search", + "user_tags", "special_field1", "special_field2", "special_field3", "graph_status", "dialog"] + + if not file_path: + # this is the default path where parser will put the txt file + file_path = self.parser_tmp_folder + + # test script for parsing txt file + try: + output_file = open(os.path.join(file_path, fn), "r", encoding="utf-8-sig",errors="ignore").read() + + except Exception as e: + logger.warning(f"warning: Parser - could not find parsing output - {file_path} - {fn}") + return [] + + # this seems to work with a few library sets, but we can probably enhance the 'splitting' + # \n marks the end of a block of text with ~28 dictionary keys + blocks = output_file.split("\n") + + output_list = [] + + for i, b in enumerate(blocks): + + # split of "\n<" will split the block into ~28 individual slices + splitter = b.split("\n<") + block_dict = {} + # it is likely redundant to have 'double loop' but it is a little extra insurance + for j, keys in enumerate(default_keys): + # iterates thru each of the default keys + match_found = -1 + for k, entries in enumerate(splitter): + + key_string = keys + ">: " + if entries.startswith(key_string): + + value = entries[len(key_string):].strip() + + # remove trailing ',' + if value.endswith(","): + value= value[:-1] + + block_dict.update({keys: value}) + match_found = 1 + break + + if match_found == -1: + # note: could not find a key - i, keys, splitter - no action required + do_nothing = 1 + + if block_dict: + if len(block_dict) == len(default_keys): + output_list.append(block_dict) + else: + logger.debug(f"Parser - convert_parsing_txt_file_to_json - potential error - " + f"parsing-to-dict conversion - lengths don't match - " + f"{len(block_dict)} - {len(default_keys)}") + + return output_list + + def parse_pdf (self, fp, write_to_db=True, save_history=True): + + """ Main PDF parser method (as of 0.2.7) - and updated further starting in version 0.3.2 - + wraps ctypes interface to call PDF parser - provides new ctypes entrypoint into PDF parser with + expanded configuration objects, and leveraging new configurations exposed in Parser construction + and Library().add_files. """ + + # adding changes for v0.3.2 - logger_level and debug_log_file + + output = [] + + write_to_filename = "pdf_parse_output_0.txt" + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + unique_doc_num = -1 + else: + write_to_db_on = 0 + unique_doc_num = int(self.file_counter) + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_pdf - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_pdf - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in /parser_history path.") + + # deprecation warning for aarch64 linux + system = platform.system().lower() + + if system == "linux": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == 'aarch64': + # re-initiating support for linux aarch64 (previously deprecated) + return self.parse_pdf_deprecated_026(fp,write_to_db=write_to_db,save_history=save_history) + + if system == "darwin": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == "x86_64": + + error_msg = ("Mac x86 detected as OS - this is not a supported platform. Support " + "was deprecated in llmware version 0.2.6 and removed in llmware version 0.3.9. " + "Options - move to Mac Metal series (e.g., M1+), back-level llmware to supported version, or " + "if urgent requirement for Mac x86, please raise ticket on github.") + + raise LLMWareException(message=error_msg) + + # end - deprecation routing + + # * function declaration for .add_pdf_main_llmware_config_new * + + # char * input_account_name + # char * input_library_name + # char * input_fp + # char * db + # char * db_uri_string + # char * db_name + # char * db_user_name + # char * db_pw + # char * input_images_fp + # int input_debug_mode + # int input_image_save_mode + # int write_to_db_on + # char * write_to_filename + # int user_blok_size + # int unique_doc_num + # int status_manager_on + # int status_manager_increment + # char * status_job_id + # int strip_header + # int table_extract + # int smart_chunking + # int max_chunk_size + # int encoding_style + # int get_header_text + # int table_grid + # int logger_level + # char *debug_log_file + + # if any issue loading module, will be captured at .get_module_pdf_parser() + _mod_pdf = Utilities().get_module_pdf_parser() + + # pdf_handler = _mod_pdf.add_pdf_main_customize_parallel + pdf_handler = _mod_pdf.add_pdf_main_llmware_config_new + + pdf_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_char_p, c_int, c_int, c_int, c_char_p, c_int,c_int,c_int,c_int,c_char_p, + c_int, c_int, c_int, c_int, c_int, c_int, c_int, + # new configs - june 14 + c_int, c_char_p) + + pdf_handler.restypes = c_int + + # prepare all of the inputs to invoke the c library + + t0 = time.time() + + # config options pulled from the Library object + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + # image_fp = self.library.image_path + image_fp = self.parser_image_folder + + if not image_fp.endswith(os.sep): + image_fp += os.sep + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + input_collection_db_path = LLMWareConfig().get_db_uri_string() + + collection_db_path_c = create_string_buffer(input_collection_db_path.encode('ascii', 'ignore')) + + # fp = passed as parameter -> this is the input file path folder containing the .PDF docs to be parsed + if not fp.endswith(os.sep): + fp += os.sep + + fp_c = create_string_buffer(fp.encode('ascii', 'ignore')) + + # debug_mode deprecated as of 0.3.1 ++ + debug_mode = self.verbose_level + + supported_options = [0, 1, 2, 3] + + if debug_mode not in supported_options: + debug_mode = 0 + + if self.get_images: + image_save = 1 # TRUE - get images + else: + image_save = 0 # FALSE - no images + + input_image_save_mode = c_int(image_save) # default - 1 = "on" | use 0 = "off" in production + + write_to_db_on_c = c_int(write_to_db_on) + write_to_filename_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + user_block_size = c_int(self.chunk_size) # standard 400-600 + + # unique_doc_num -> if <0: interpret as "OFF" ... if >=0 then use and increment doc_id directly + # unique_doc_num = -1 + unique_doc_num_c = c_int(unique_doc_num) + + # db credentials + db_user_name = self.collection_db_username + db_user_name_c = create_string_buffer(db_user_name.encode('ascii', 'ignore')) + + db_pw = self.collection_db_password + db_pw_c = create_string_buffer(db_pw.encode('ascii', 'ignore')) + + db = LLMWareConfig.get_config("collection_db") + + db = create_string_buffer(db.encode('ascii','ignore')) + db_name = account_name + + status_manager_on = c_int(1) + status_manager_increment = c_int(10) + status_job_id = create_string_buffer("1".encode('ascii','ignore')) + + # defaults to 0 + if self.strip_header: + strip_header = c_int(1) + else: + strip_header = c_int(0) + + if self.get_tables: + table_extract = c_int(1) + else: + table_extract = c_int(0) + + smart_chunking = c_int(self.smart_chunking) + + # by default - 1 = get header text || turn off = 0 + if self.get_header_text: + get_header_text = c_int(1) + else: + get_header_text = c_int(0) + + if self.table_grid: + table_grid = c_int(1) + else: + table_grid = c_int(0) + + max_chunk_size = c_int(self.max_chunk_size) + + if self.encoding == "ascii": + encoding_style = c_int(0) + elif self.encoding == "utf-8": + encoding_style = c_int(2) + elif self.encoding == "latin-1": + encoding_style = c_int(1) + else: + encoding_style = c_int(0) + + if self.use_logging_file: + + # parsers use code of 60 to indicate log_to_file stream rather than stdout + input_debug_mode = c_int(60) + else: + input_debug_mode = c_int(0) + + # + # * main call to pdf library * + # + + logger.info("Parser - parse_pdf - start parsing of PDF Documents...") + + logger_level = c_int(self.logger_level) + dlf_fp = os.path.join(self.parser_folder, self.parser_log_name) + debug_log_file = create_string_buffer(dlf_fp.encode('ascii', 'ignore')) + + pages_created = pdf_handler(account_name, library_name, fp_c, db, collection_db_path_c, db_name, + db_user_name_c, db_pw_c, + image_fp_c, + input_debug_mode, input_image_save_mode, write_to_db_on_c, + write_to_filename_c, user_block_size, unique_doc_num_c, + status_manager_on, status_manager_increment, status_job_id, + strip_header, table_extract, smart_chunking, max_chunk_size, + encoding_style, get_header_text, table_grid, + # new params added in 0.3.2 + logger_level, debug_log_file + ) + + logger.info(f"Parser - parse_pdf - completed parsing of pdf documents - time taken: {time.time()-t0}") + + if write_to_db_on == 0: + # package up results in Parser State + parser_output = self.convert_parsing_txt_file_to_json(self.parser_image_folder, write_to_filename) + if len(parser_output) > 0: + last_entry = parser_output[-1] + last_doc_id = last_entry["doc_ID"] + + self.file_counter = int(last_doc_id) + + logger.info(f"Parser - parse_pdf - adding new entries to parser output state - {len(parser_output)}") + + self.parser_output += parser_output + output += parser_output + + if save_history: + ParserState().save_parser_output(self.parser_job_id, parser_output) + + return output + + def parse_office(self, input_fp, write_to_db=True, save_history=True): + + """ Primary method interface into Office parser with more configuration options - expanded most + recently in version 0.3.2 """ + + output = [] + + # used internally by parser to capture text + write_to_filename = "office_parser_output_0.txt" + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + unique_doc_num = -1 + else: + write_to_db_on = 0 + unique_doc_num = int(self.file_counter) + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_office - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in Parser /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_office - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in Library /images path.") + + system = platform.system().lower() + + if system == "linux": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == 'aarch64': + # re-initiating support for linux aarch64 + return self.parse_office_deprecated_027(input_fp,write_to_db=write_to_db,save_history=save_history) + + if system == "darwin": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == "x86_64": + + error_msg = ("Mac x86 detected as OS - this is not a supported platform. Support " + "was deprecated in llmware version 0.2.6 and removed in llmware version 0.3.9. " + "Options - move to Mac Metal (M1+), back-level llmware to supported version, or " + "if urgent requirement for Mac x86, please raise ticket on github.") + + raise LLMWareException(message=error_msg) + + # end - deprecation routing + + # designed for bulk upload of office parse into library structure + + if not input_fp.endswith(os.sep): + input_fp += os.sep + + office_fp = input_fp + + workspace_fp = os.path.join(self.parser_tmp_folder, "office_tmp" + os.sep) + + if not os.path.exists(workspace_fp): + os.mkdir(workspace_fp) + os.chmod(workspace_fp, 0o777) + + # start timing track for parsing job + t0 = time.time() + + # only one tmp work folder used currently - can consolidate over time + for z in range(0, 5): + + if os.path.exists(os.path.join(workspace_fp, str(z))): + shutil.rmtree(os.path.join(workspace_fp, str(z)), ignore_errors=True) + + if not os.path.exists(os.path.join(workspace_fp, str(z))): + os.mkdir(os.path.join(workspace_fp, str(z))) + os.chmod(os.path.join(workspace_fp, str(z)), 0o777) + + # end -initialize workspace + + # if any issue loading module, will be captured at .get_module_office_parser() + _mod = Utilities().get_module_office_parser() + + main_handler = _mod.add_files_main_llmware_opt_full + + # * function declaration for add_files_main_llmware_opt_full * + + # char * input_account_name + # char * input_library_name + # char * input_fp + # char * workspace_fp + # char * db + # char * db_uri_string + # char * db_name + # char * db_user_name + # char * db_pw + # char * image_fp + # int input_debug_mode + # int write_to_db_on + # char * write_to_filename + # int unique_doc_num + # int user_blok_size + # int status_manager_on + # int status_manager_increment + # char * status_job_id + # int strip_header + # int table_extract + # int smart_chunking + # int max_chunk_size + # int encoding_style + # int get_header_text + # int table_grid + # int save_images + # int logger_level + # char* debug_file + + main_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_int, c_int, c_char_p, c_int, c_int, c_int, c_int, + c_char_p, c_int, c_int, c_int, c_int, c_int, c_int, + c_int, c_int, c_int, c_char_p) + + main_handler.restype = c_int + + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + fp_c = create_string_buffer(office_fp.encode('ascii', 'ignore')) + workspace_fp_c = create_string_buffer(workspace_fp.encode('ascii', 'ignore')) + + # debug_mode deprecated as of 0.3.1++ + debug_mode = self.verbose_level + + supported_options = [0, 1, 2, 3] + + if debug_mode not in supported_options: + debug_mode = 0 + + debug_mode_c = c_int(debug_mode) + + image_fp = self.parser_image_folder + if not image_fp.endswith(os.sep): + image_fp += os.sep + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + # get db uri string + input_collection_db_path = LLMWareConfig().get_db_uri_string() + collection_db_path_c = create_string_buffer(input_collection_db_path.encode('ascii', 'ignore')) + + write_to_db_on_c = c_int(write_to_db_on) + + write_to_fn_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + # unique_doc_num is key parameter - if <0: will pull from incremental db, if >=0, then will start at this value + # unique_doc_num = -1 + unique_doc_num_c = c_int(unique_doc_num) + + # pull target block size from library parameters + user_block_size_c = c_int(self.chunk_size) + + # db credentials + db_user_name = self.collection_db_username + db_user_name_c = create_string_buffer(db_user_name.encode('ascii', 'ignore')) + + db_pw = self.collection_db_password + db_pw_c = create_string_buffer(db_pw.encode('ascii', 'ignore')) + + db = LLMWareConfig.get_config("collection_db") + + db = create_string_buffer(db.encode('ascii', 'ignore')) + db_name = account_name + + status_manager_on_c = c_int(1) + status_manager_increment_c = c_int(10) + status_job_id_c = create_string_buffer("1".encode('ascii', 'ignore')) + + # defaults to 0 + if self.strip_header: + strip_header = c_int(1) + else: + strip_header = c_int(0) + + if self.get_tables: + table_extract = c_int(1) + else: + table_extract = c_int(0) + + smart_chunking = c_int(self.smart_chunking) + + # by default - 1 = get header text || turn off = 0 + if self.get_header_text: + get_header_text = c_int(1) + else: + get_header_text = c_int(0) + + if self.table_grid: + table_grid = c_int(1) + else: + table_grid = c_int(0) + + if self.encoding == "ascii": + encoding_style = c_int(0) + elif self.encoding == "utf-8": + encoding_style = c_int(2) + else: + encoding_style = c_int(2) + + max_chunk_size = c_int(self.max_chunk_size) + + if self.get_images: + save_images = c_int(1) # TRUE - get images + else: + save_images = c_int(0) # FALSE - no images + + logger.info("Parser - parse_office - start parsing of office documents...") + + if self.use_logging_file: + input_debug_mode = c_int(60) + else: + input_debug_mode = c_int(0) + + logger_level = c_int(self.logger_level) + + dlf_fp = os.path.join(self.parser_folder, self.parser_log_name) + + debug_log_file = create_string_buffer(dlf_fp.encode('ascii', 'ignore')) + + pages_created = main_handler(account_name, library_name, fp_c, workspace_fp_c, + db, collection_db_path_c, db_name, db_user_name_c, db_pw_c, + image_fp_c, + input_debug_mode, write_to_db_on_c, write_to_fn_c, unique_doc_num_c, + user_block_size_c, status_manager_on_c, status_manager_increment_c, + status_job_id_c, strip_header, table_extract, smart_chunking, + max_chunk_size, encoding_style, get_header_text, table_grid, + save_images, logger_level, debug_log_file) + + logger.info(f"Parser - parse_office - completed parsing of office documents - time taken: {time.time()-t0}") + + if write_to_db_on == 0: + # package up results in Parser State + parser_output = self.convert_parsing_txt_file_to_json(self.parser_image_folder, write_to_filename) + if len(parser_output) > 0: + last_entry = parser_output[-1] + last_doc_id = last_entry["doc_ID"] + + self.file_counter = int(last_doc_id) + + self.parser_output += parser_output + output += parser_output + + if save_history: + # save parser state + ParserState().save_parser_output(self.parser_job_id, parser_output) + + return output + + def parse_text(self, input_fp, write_to_db=True, save_history=True, dupe_check=False,copy_to_library=False, + text_chunk_size=None, key_list=None, interpret_as_table=False,delimiter=",", separator="\n", + batch_size=1, encoding="utf-8-sig", errors="ignore"): + + """ Main entry point to parser for .txt, .csv, .json, .jsonl, .tsv and .md files """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_text - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_text - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the file " + f"in /parser_history path.") + + # set counters + blocks_created = 0 + docs_added = 0 + pages_added = 0 + content_type = "text" + + for file in os.listdir(input_fp): + + # by default, will process all files with text file extensions + go_ahead = True + + if dupe_check: + + # basic_library_duplicate_check returns TRUE if it finds the file + if self.basic_library_duplicate_check(file): + go_ahead = False + + if go_ahead: + + text_output = [] + # increment and get new doc_id + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + logger.info(f"Parser - parse_text file - processing - {file}") + + file_type = file.split(".")[-1] + + # sub-routing by type of text file to appropriate handler + + if file_type.lower() in ["txt", "md"]: + # will parse as text + text_output = TextParser(self,text_chunk_size=text_chunk_size).text_file_handler (input_fp, file) + content_type = "text" + file_type = "txt" + + if file_type.lower() in ["csv", "tsv"]: + + if file_type.lower() == "tsv": + delimiter= "\t" + + text_output = ( TextParser(self,text_chunk_size=text_chunk_size). + csv_file_handler(input_fp, file, interpret_as_table=interpret_as_table, + delimiter=delimiter, batch_size=batch_size, encoding=encoding, + errors=errors) ) + + content_type = "text" + file_type = file_type.lower() + if interpret_as_table: + content_type = "table" + + if file_type.lower() in ["json","jsonl"]: + # will parse each line item as separate entry + + interpret_as_table=False + if not key_list: + key_list = ["text"] + text_output = TextParser(self).jsonl_file_handler(input_fp,file, + key_list=key_list, + interpret_as_table=interpret_as_table, + separator="\n") + content_type = "text" + file_type = "jsonl" + if interpret_as_table: + content_type = "table" + + # consolidate into single function - breaking down output rows + + if write_to_db_on == 1: + new_output, new_blocks, new_pages = self._write_output_to_db(text_output, file, + content_type=content_type, + file_type=file_type) + else: + new_output, new_blocks, new_pages = self._write_output_to_dict(text_output,file, + content_type=content_type, + file_type=file_type) + + # will pass output_blocks as return value + output += new_output + + docs_added += 1 + blocks_created += new_blocks + pages_added += new_pages + + # update overall library counter at end of parsing + + if len(output) > 0: + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=docs_added,added_blocks=blocks_created, + added_images=0, added_pages=pages_added) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + if copy_to_library: + self.uploads(input_fp) + + return output + + def parse_pdf_by_ocr_images(self, input_fp, write_to_db=True, save_history=True, + dupe_check=False,copy_to_library=False): + + """ Alternative PDF parser option for scanned 'image-based' PDFs where digital parsing is not an option. """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_text - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_text - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in /parser_history path.") + + # set counters + blocks_added = 0 + docs_added = 0 + pages_added = 0 + + content_type = "text" + + for file in os.listdir(input_fp): + + # by default, will process all files with text file extensions + go_ahead = True + + if dupe_check: + + # basic_library_duplicate_check returns TRUE if it finds the file + if self.basic_library_duplicate_check(file): + go_ahead = False + + if go_ahead: + + ext = file.split(".")[-1] + if ext == "pdf": + + doc_fn = Utilities().secure_filename(file) + + # get new doc_ID number + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + docs_added += 1 + + output_by_page = ImageParser(self).process_pdf_by_ocr(input_fp, file) + + for j, blocks in enumerate(output_by_page): + + if write_to_db_on == 1: + new_output, new_blocks, _ = self._write_output_to_db(blocks,doc_fn,page_num=(j+1)) + else: + new_output, new_blocks, _ = self._write_output_to_dict(blocks,doc_fn,page_num=(j+1)) + + output += new_output + blocks_added += new_blocks + pages_added += 1 + + logger.info(f"Parser - parse_pdf_by_ocr_images - writing doc - page - " + f"{file} - {j} - {len(blocks)}") + + # update overall library counter at end of parsing + + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=docs_added,added_blocks=blocks_added, + added_images=0, added_pages=pages_added) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + if copy_to_library: + self.uploads(input_fp) + + return output + + def _write_output_to_db(self, output, file, content_type="text", file_type="text",page_num=1): + + """ Internal utility for preparing parser output to write to DB. """ + + db_record_output = [] + + # trackers + blocks_added = 0 + pages_added = 0 + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + counter = 0 + + for entries in output: + + if content_type == "text": + # table entry = "" [7] + new_entry = (content_type, file_type, (page_num, 0), counter, "", "", file, "", entries, "", + "", entries, entries, "", entries, "", "", "", "", "") + else: + # could be table if csv file -> in this case, keep both text [11] and table [7] + if not isinstance(entries,str): + entries = str(entries) + + new_entry = (content_type, file_type, (page_num, 0), counter, "", "", file, entries, entries, "", + "", entries, entries, "", entries, "", "", "", "", "") + + counter += 1 + + new_db_entry = self.add_create_new_record(self.library,new_entry, meta, coords_dict) + db_record_output.append(new_db_entry) + + blocks_added += 1 + self.library.block_ID += 1 + + # need to adapt potentially for longer text files + pages_added = 1 + + return db_record_output, blocks_added, pages_added + + def _write_output_to_dict(self, wp_output, input_fn, content_type="text", file_type="text", page_num=1): + + """ Internal utility for preparing parser output to dictionary. """ + + output = [] + # consolidate output + counter = 0 + blocks_added = 0 + pages_added = 0 + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + for j, blocks in enumerate(wp_output): + + if content_type == "text": + new_entry = ("text", file_type, (page_num, 0), counter, "", "", input_fn, "", blocks, "", + "", blocks, blocks, "", blocks, "", "", "", "", "") + else: + # could be table if csv file -> in this case, keep both text [11] and table [7] + new_entry = ("table", file_type, (page_num, 0), counter, "", "", input_fn, blocks, blocks, "", + "", blocks, blocks, "", blocks, "", "", "", "", "") + + # creates a single 'unbound' parsing output dict -> no storage + parsing_output_dict = self.create_one_parsing_output_dict(counter, + new_entry, meta, coords_dict, + dialog_value="false") + + output.append(parsing_output_dict) + blocks_added += 1 + + pages_added = 1 + + self.parser_output += output + + return output, blocks_added, pages_added + + def add_create_new_record(self, library, new_entry, meta, coords_dict,dialog_value="false", + write_to_db=True, custom_doc_id=None, custom_block_id=None): + + """ Main 'write' method of new parser text chunk for python-based parsers to write to DB. """ + + # assumes that new_entry is packaged in individual handler + # objective is to keep one single place where new entry gets loaded into db + # ensure consistency of db data model + + if custom_doc_id: + new_doc_id = custom_doc_id + else: + new_doc_id = library.doc_ID + + if custom_block_id: + new_block_id = custom_block_id + else: + new_block_id = library.block_ID + + time_stamp = Utilities().get_current_time_now() + + new_entry = { + "block_ID": new_block_id, # note - needs caution + "doc_ID": new_doc_id, # note - needs caution + "content_type": new_entry[0], + "file_type": new_entry[1], + "master_index": new_entry[2][0], + # change from [1:] to [1] + "master_index2": new_entry[2][1], + "coords_x": coords_dict["coords_x"], + "coords_y": coords_dict["coords_y"], + "coords_cx": coords_dict["coords_cx"], + "coords_cy": coords_dict["coords_cy"], + "author_or_speaker": meta["author"], + "modified_date": meta["modified_date"], + "created_date": meta["created_date"], + "creator_tool": meta["creator_tool"], + "added_to_collection": time_stamp, + "file_source": new_entry[6], + "table": new_entry[7], + "external_files": new_entry[10], + "text": new_entry[11], + "header_text": new_entry[13], + "text_search": new_entry[14], + "user_tags": new_entry[15], + "special_field1": new_entry[17], + "special_field2": new_entry[18], + "special_field3": new_entry[19], + "graph_status": "false", + "dialog": dialog_value, + "embedding_flags": {} + } + + if write_to_db: + # registry_id = library.collection.insert_one(new_entry).inserted_id + registry_id = CollectionWriter(library.library_name, + account_name=library.account_name).write_new_parsing_record(new_entry) + + return new_entry + + def create_one_parsing_output_dict(self, block_id,new_entry, meta, coords_dict,dialog_value="false"): + + """ Main method to prepare a new text chunk parser output for python-based parser as dictionary. """ + + # Mirrors the data structure in "self.add_create_new_record" + # --does not write_to_db or storage + # --does not assume that there is a library index + # --creates one parsing output dict that can be used and stored for any purpose (outside of library) + + # Note: expects explicit passing of a block_id and doc_id as reference numbers + + time_stamp = Utilities().get_current_time_now() + + new_entry = { + "block_ID": block_id, + "doc_ID": self.file_counter, + "content_type": new_entry[0], + "file_type": new_entry[1], + "master_index": new_entry[2][0], + # change from [1:] to [1] + "master_index2": new_entry[2][1], + "coords_x": coords_dict["coords_x"], + "coords_y": coords_dict["coords_y"], + "coords_cx": coords_dict["coords_cx"], + "coords_cy": coords_dict["coords_cy"], + "author_or_speaker": meta["author"], + "modified_date": meta["modified_date"], + "created_date": meta["created_date"], + "creator_tool": meta["creator_tool"], + "added_to_collection": time_stamp, + "file_source": new_entry[6], + "table": new_entry[7], + "external_files": new_entry[10], + "text": new_entry[11], + "header_text": new_entry[13], + "text_search": new_entry[14], + "user_tags": new_entry[15], + "special_field1": new_entry[17], + "special_field2": new_entry[18], + "special_field3": new_entry[19], + "graph_status": "false", + "dialog": dialog_value, + "embedding_flags": "" + } + + return new_entry + + def parse_wiki(self, topic_list, write_to_db=True, save_history=False, target_results=10): + + """ Main entry point to parse a Wikipedia article. """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_text - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_text - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in /parser_history path.") + + # set counters + blocks_added = 0 + docs_added = 0 + pages_added = 0 + + for i, topic in enumerate(topic_list): + + fn = "wiki-topic-" + Utilities().secure_filename(topic) + ".txt" + + logger.info(f"Parser - parse_wiki - {topic} - {fn}") + + # increment and get new doc_id + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + # topic_results = {"search_results": topic_query_results, "articles": articles_output, + # "text_chunks": text_chunks} + + topic_results = WikiParser(self).add_wiki_topic(topic, target_results=target_results) + + wp_output = topic_results["text_chunks"] + + if write_to_db_on == 1: + new_output, new_blocks, new_pages = self._write_output_to_db(wp_output, fn, content_type="text", + file_type="wiki") + + else: + new_output, new_blocks, new_pages = self._write_output_to_dict(wp_output,fn, content_type="text", + file_type="wiki") + output += new_output + + docs_added += 1 + blocks_added += new_blocks + pages_added += new_pages + + for i, articles in enumerate(topic_results["articles"]): + + # need to copy into library_copy path + if self.library: + upload_fp = self.library.file_copy_path + else: + upload_fp = self.parser_tmp_folder + + # save as the article title now + article_txt = articles["title"]+".txt" + safe_name = self.prep_filename(article_txt) + + art = open(os.path.join(upload_fp,safe_name), "w", encoding='utf-8') + art.write(articles["text"]) + art.close() + + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=docs_added, added_blocks=blocks_added, + added_images=0, added_pages=pages_added) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_image(self, input_folder, write_to_db=True, save_history=True, dupe_check=False,copy_to_library=False): + + """ Main entry point for OCR based parsing of image files. """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_text - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_text - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in /parser_history path.") + + # set counters + blocks_added = 0 + docs_added = 0 + pages_added = 0 + + for file in os.listdir(input_folder): + + # by default, will process all files with text file extensions + go_ahead = True + + if dupe_check: + + # basic_library_duplicate_check returns TRUE if it finds the file + if self.basic_library_duplicate_check(file): + go_ahead = False + + if go_ahead: + + # increment and get new doc_id + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + ip_output = ImageParser(self).process_ocr(input_folder, file) + + if write_to_db_on == 1: + new_output, new_blocks, new_pages = self._write_output_to_db(ip_output,file,content_type="text", + file_type="ocr") + else: + new_output, new_blocks, new_pages = self._write_output_to_dict(ip_output,file, content_type="text", + file_type="ocr") + # return output value in either case + output += new_output + + docs_added += 1 + blocks_added += new_blocks + pages_added += new_pages + + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=docs_added, added_blocks=blocks_added, + added_images=0, added_pages=pages_added) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + if copy_to_library: + self.uploads(input_folder) + + return output + + def parse_voice(self, input_folder, write_to_db=True, save_history=True, dupe_check=False,copy_to_library=False, + chunk_by_segment=True, remove_segment_markers=True, real_time_progress=True): + + """ Main entry point for parsing voice wav files. """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_voice - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_voice - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in /parser_history path.") + + # set counters + blocks_added = 0 + docs_added = 0 + pages_added = 0 + + for file in os.listdir(input_folder): + + # by default, will process all files with text file extensions + go_ahead = True + + if dupe_check: + + # basic_library_duplicate_check returns TRUE if it finds the file + if self.basic_library_duplicate_check(file): + go_ahead = False + + if go_ahead: + + # increment and get new doc_id + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + logger.info(f"Parser - parse_voice file - processing - {file}") + + vp_output = VoiceParser(self, + chunk_size=self.chunk_size, + max_chunk_size=self.max_chunk_size, + chunk_by_segment=chunk_by_segment, + remove_segment_markers=remove_segment_markers, + real_time_progress=real_time_progress).add_voice_file(input_folder, file) + + if not chunk_by_segment: + text_chunks_only = [] + for chunks in vp_output: + text_chunks_only.append(chunks["text"]) + + if write_to_db_on == 1: + new_output, new_blocks, new_pages = self._write_output_to_db(text_chunks_only, file, + content_type="text", + file_type="voice-wav") + else: + new_output, new_blocks, new_pages = self._write_output_to_dict(text_chunks_only, file, + content_type="text", + file_type="voice-wav") + + output += new_output + docs_added += 1 + blocks_added += new_blocks + pages_added += new_pages + self.file_counter += 1 + + else: + + for i, blocks in enumerate(vp_output): + + # iterate thru each block -> add to metadata + speaker_name = blocks["speaker"] + + meta = {"author": speaker_name, "modified_date": "", "created_date": "", "creator_tool": ""} + + coords_dict = {"coords_x": blocks["start_time"], "coords_y": blocks["end_time"], + "coords_cx": blocks["start_segment"], "coords_cy": blocks["end_segment"]} + + text_entry = blocks["text"] + + format_type = "voice-wav" + + new_entry = ("text", format_type, (1, 0), i, "", "", file, + "", text_entry, "", "", text_entry, text_entry, "", text_entry, + "", "", "", "", "") + + #TODO: adding dialog and diarization roles in speech parsing + + if write_to_db_on == 1: + entry_output = self.add_create_new_record(self.library, new_entry, meta, coords_dict, + dialog_value="false") + self.library.block_ID += 1 + else: + entry_output = self.create_one_parsing_output_dict(i,new_entry,meta,coords_dict, + dialog_value="false") + self.parser_output.append(entry_output) + + # return output in either case + output.append(entry_output) + + blocks_added += len(vp_output) + pages_added += 0 + docs_added += 1 + self.file_counter += 1 + + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=docs_added, added_blocks=blocks_added, + added_images=0, added_pages=pages_added) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + if copy_to_library: + self.uploads(input_folder) + + return output + + def parse_dialog(self, input_folder, write_to_db=True, save_history=True, dupe_check=False,copy_to_library=True): + + """ Main entry point for parsing AWS dialog transcripts. """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_dialog - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_dialog - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place " + f"the file in /parser_history path.") + + # set counters + conversation_turns = 0 + dialog_transcripts_added = 0 + counter = 0 + + for file in os.listdir(input_folder): + + # by default, will process all files with text file extensions + go_ahead = True + + if dupe_check: + + # basic_library_duplicate_check returns TRUE if it finds the file + if self.basic_library_duplicate_check(file): + go_ahead = False + + if go_ahead: + + if file.endswith(".json"): + + # increment and get new doc_id + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + logger.info(f"Parser - parse_dialog - dialog file - {file}") + + dp_parse_output = DialogParser(self).parse_aws_json_file_format(input_folder, file) + + block_id = 0 + + for i, blocks in enumerate(dp_parse_output): + + logger.debug(f"Parser - parse_dialog - dialog turn - {i} {blocks}") + + # iterate thru each block -> add to metadata + speaker_name = blocks["speaker_name"] + + meta = {"author": speaker_name, "modified_date": "", "created_date": "", "creator_tool": ""} + + coords_dict = {"coords_x": blocks["start_time"], "coords_y": blocks["stop_time"], + "coords_cx": 0, "coords_cy": 0} + + text_entry = blocks["text"] + + # conforming file format with full path of dialog intake path + + format_type = "aws_json" + + new_entry = ("text", format_type, (1, 0), counter, "", "", input_folder + file, + text_entry, text_entry, "", "", text_entry, text_entry, "", text_entry, + "", "", "", "", "") + + counter += 1 + dialog_transcripts_added += 1 + conversation_turns += 1 + + if write_to_db_on == 1: + entry_output = self.add_create_new_record(self.library, new_entry, meta, coords_dict, + dialog_value="true") + self.library.block_ID += 1 + else: + entry_output = self.create_one_parsing_output_dict(block_id,new_entry,meta,coords_dict, + dialog_value="true") + block_id += 1 + self.parser_output.append(entry_output) + + # return output in either case + output.append(entry_output) + + pages_added = dialog_transcripts_added + + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=dialog_transcripts_added, + added_blocks=conversation_turns, + added_images=0, + added_pages=pages_added) + + # by default copies transcripts to upload folder + if copy_to_library: + self.uploads(input_folder) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_website(self, url_base, write_to_db=True, save_history=True, get_links=True, max_links=10): + + """ Main entrypoint for parsing a website. """ + + output = [] + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + else: + write_to_db_on = 0 + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_website - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_website - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in /parser_history path.") + + local_work_folder = self.parser_tmp_folder + + if not os.path.exists(local_work_folder): + os.mkdir(local_work_folder) + + self.website_work_folder = os.path.join(local_work_folder, "process_website" + os.sep) + + # start clean + if os.path.exists(self.website_work_folder): + shutil.rmtree(self.website_work_folder, ignore_errors=True) + os.mkdir(self.website_work_folder) + + # iterative parse thru website to follow links enabled + + website = WebSiteParser(url_base, reset_img_folder=True, local_file_path=self.website_work_folder) + + if website.success_code == 1: + + # increment and get new doc_id + if write_to_db_on == 1: + self.library.doc_ID = self.library.get_and_increment_doc_id() + + entries, img_counter = website.website_main_processor(website.image_counter, + output_index=False) + + # if get_links, then pursue internal links and 'add' to indexed output gathered + + if get_links: + + if len(website.internal_links) > 0: + + max_links = min(len(website.internal_links), max_links) + + for z in range(0, max_links): + + logger.debug(f"Parser - parse_website - iterate - " + f"child site link - {z} - {url_base} - {website.internal_links[z]}") + + child_site = WebSiteParser(url_base + website.internal_links[z], reset_img_folder=False, + local_file_path=self.website_work_folder) + + if child_site.success_code == 1: + new_child_entries, img_counter = child_site.website_main_processor(img_counter, + output_index=False) + + for c in range(0, len(child_site.core_index)): + website.core_index.append(child_site.core_index[c]) + + # write parser output to storage + + entries_created = 0 + images_created = 0 + running_links = "" + file_type = "html" + + file_source = str(random.randint(100000, 999999)) + "_" + website.url_main.split(".")[-2] + ".html" + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + # prep loop - consolidate links with text or image + for z in range(0, len(website.core_index)): + + """ + # core index entry is dictionary + entry = {"content_type": entry_type, + "text": text, + "image": {"image_name": img_name, "image_url": img_url}, + "link": {"link_type": link_type, "link": link}, + "master_index": master_index, + "last_header": last_header} + """ + + content_type = website.core_index[z]["content_type"] + + if content_type == "link": + link_type = website.core_index[z]["link"]["link_type"] + + if link_type == "internal": + # attach internal links to last piece of text or image + running_links += website.core_index[z]["link"]["link"] + " , " + + if content_type == "text" or content_type == "image": + # close out last entry & start new one + save_entry = 1 + text1_core = website.core_index[z]["text"] + if not text1_core: + text1_core = website.core_index[z]["last_header"] + + # no tables currently extracted in website parser + content1_core = "" + + text3_format = website.core_index[z]["last_header"] + text2_spatial = running_links + links = running_links + running_links = "" + + master_index = (entries_created, 0) + coords = master_index + user_tags = [] + external_files = "" + + if content_type == "image": + + fp_tmp = self.website_work_folder + image_num = website.core_index[z]["image"]["image_name"] + + if self.library: + doc_id = self.library.doc_ID + save_file_path = self.library.image_path + else: + doc_id = self.file_counter + save_file_path = self.parser_image_folder + + new_image_name, created = website._save_image_website(fp_tmp, image_num, doc_id, save_file_path) + + images_created += 1 + external_files = new_image_name + + if not text1_core: + # take adjacent header_text, if no text linked to image + text1_core = text3_format + + new_entry = (content_type, file_type, master_index, "", "", "", + file_source, content1_core,"","", external_files, text1_core,text2_spatial, + text3_format,text1_core, user_tags,links,"","" ,"") + + if write_to_db_on == 1: + entry_output = self.add_create_new_record(self.library, new_entry,meta,coords_dict) + else: + entry_output = self.create_one_parsing_output_dict(entries_created, + new_entry,meta,coords_dict, + dialog_value="false") + self.parser_output.append(entry_output) + output.append(entry_output) + entries_created += 1 + + # once done with all of the record updates- update the master counters + # need to save new block_ID & new doc_ID + docs_created = 1 + self.file_counter += 1 + + if write_to_db_on == 1: + dummy = self.library.set_incremental_docs_blocks_images(added_docs=docs_created, + added_blocks=entries_created, + added_images=images_created, + added_pages=1) + + # upload website_file + fp_tmp = os.path.join(local_work_folder, "process_website" + os.sep) + + website_name = "my_website.html" + + # apply secure filename to remove any extra "/" + secure_url_name = Utilities().secure_filename(website.url_main.split(".")[-2]) + + out_name = str(random.randint(100000, 999999)) + "_" + secure_url_name + ".html" + + if self.library: + upload_fp = self.library.file_copy_path + else: + upload_fp = self.parser_tmp_folder + + shutil.copy(os.path.join(fp_tmp,website_name), os.path.join(upload_fp, out_name)) + + if save_history and write_to_db_on == 0: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def uploads(self, tmp_dir, overwrite=False): + + """ Utility method that handles 'uploads' of input files into library structure. """ + + # designed for upload of input files into library structure + + if not self.library: + logger.error("Parser - uploads is designed for connecting files into library - " + "no library selected - to use, create Parser with library loaded, e.g., " + "Parser(library=my_library)") + return -1 + + upload_fp = self.library.file_copy_path + library_files = os.listdir(upload_fp) + files = os.listdir(tmp_dir) + for x in range(0, len(files)): + safe_name = self.prep_filename(files[x]) + + # exclude any folders + if not os.path.isdir(os.path.join(tmp_dir,files[x])): + + # will not over-write an existing file unless overwrite flag set + if overwrite or files[x] not in library_files: + shutil.copy(os.path.join(tmp_dir, files[x]), os.path.join(upload_fp, files[x])) + + return len(files) + + def prep_filename(self, fn, secure_name=True, prepend_string=None, postpend_string=None, max_len=None): + + """ Utility function to prepare 'safe' filenames """ + + fn_out = fn + + # default - apply basic secure name, e.g., remove / and insert _ + if secure_name: + fn_out= Utilities().secure_filename(fn) + + # if requested prepend or postpend + if prepend_string: + fn_out= prepend_string + fn_out + + if postpend_string: + fn_base, ext = fn_out.split(".") + fn_out = fn_base + postpend_string + ext + + # if max len applied + if max_len: + if len(fn_out) > max_len: + fn_base, ext = fn_out.split(".") + fn_out = fn_base[0:max_len-len(ext)] + ext + + return fn_out + + def input_ingestion_comparison (self, file_list): + + """ Compares input with parsed output to identify any rejected files. """ + + # shortcut if file_list is just empty + if len(file_list) < 1: + return [],[] + + # simple approach - compares input file_list from ingestion 'work_order' with state of library collection + # --if input file found, then added to 'found_list' -> else, added to 'not_found_list' + + if not self.library: + logger.error("Parser - input_ingestion_comparison is designed for bulk parsing of files " + "into library - no library selected - to use, create Parser with library loaded, e.g., " + "Parser(library=my_library)") + return -1 + + found_list = [] + + doc_fn_raw_list = CollectionRetrieval(self.library_name, + account_name=self.account_name).get_distinct_list("file_source") + + for i, file in enumerate(doc_fn_raw_list): + + if file.split(os.sep)[-1] in file_list: + # excludes zip files that have been unzipped into core files in the parsing proces + found_list.append(file.split(os.sep)[-1]) + + # if found_list is equal length of file_list we don't need to look any further + if len(found_list) == len(file_list): + break + + not_found_list = list(set(file_list) - set(found_list)) + + # will strip any .zip files from rejected list, since the individual files are dynamically extracted + # and parsed, and if there is an error opening the zip it is raised as an exception + + ex_zip_nf_list = [] + for f in not_found_list: + if not f.endswith(".zip"): + ex_zip_nf_list.append(f) + + return found_list, ex_zip_nf_list + + def input_ingestion_comparison_from_parser_state (self, file_list): + + """ Compares input with parsed output to identify any rejected files. """ + + # simple approach - compares input file_list from ingestion 'work_order' with state of library collection + # --if input file found, then added to 'found_list' -> else, added to 'not_found_list' + + doc_fn_out = [] + + for i, doc_fn in enumerate(self.parser_output): + if "file_source" in doc_fn: + if doc_fn["file_source"] not in doc_fn_out: + doc_fn_out.append(doc_fn["file_source"]) + + found_list = [] + not_found_list = [] + + for i, input_file in enumerate(file_list): + found_file = -1 + for j, ingested_file in enumerate(doc_fn_out): + + # need to confirm 'symmetrical' transformations, e.g., secure filename and any prepend/postpend + if input_file == ingested_file: + found_file = 1 + found_list.append(input_file) + break + if found_file == -1: + not_found_list.append(input_file) + + return found_list, not_found_list + + def parse_one (self, fp, fn, save_history=True): + + """ Parse one 'ad hoc' 'unbound' parsing of a single document in memory -> no library required """ + + # check that path exists + if not os.path.exists(os.path.join(fp, fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(fp,fn)}") + + output = [] + + ext = fn.split(".")[-1].lower() + + if ext == "pdf": + output = self.parse_one_pdf(fp, fn, save_history=False) + + if ext in self.office_types: + output = self.parse_one_office(fp, fn, save_history=False) + + if ext in self.text_types: + output = self.parse_one_text(fp, fn, save_history=False) + + if ext in self.voice_types: + output = self.parse_one_voice(fp, fn, save_history=False) + + # no history saved by the individual parsers, as it will be saved below + if save_history: + if output: + ParserState().save_parser_output(self.parser_job_id, output) + + if not output: + logger.warning(f"Parser - parse_one - no content parsed from document - {fn}") + + return output + + def parse_one_office (self, fp, fn, save_history=True): + + """ Parse one office document at selected file path and file name. """ + + # Designed for 'ad hoc' and 'unbound' quick parse of a single office document with no storage + # -- output provided as list of Dicts in memory with same structure as parsing output + # -- updated with expanded configuration options for logging (0.3.2+) + + # check that path exists + if not os.path.exists(os.path.join(fp, fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(fp, fn)}") + + system = platform.system().lower() + + if system == "linux": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == 'aarch64': + # re-initiating support for linux aarch platform + return self.parse_one_office_deprecated_031_no_opts(fp,fn,save_history=save_history) + + if system == "darwin": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == "x86_64": + + error_msg = ("Mac x86 detected as OS - this is not a supported platform. Support " + "was deprecated in llmware version 0.2.6 and removed in llmware version 0.3.9. " + "Options - move to Mac Metal (M1+), back-level llmware to supported version, or " + "if urgent requirement for Mac x86, please raise ticket on github.") + + raise LLMWareException(message=error_msg) + + # end - deprecation routing + + workspace_fp = self.parser_tmp_folder + + if not os.path.exists(workspace_fp): + os.mkdir(workspace_fp) + os.chmod(workspace_fp, 0o777) + + # safety check - will need to improve + expand for supporting windows path + if not workspace_fp.endswith(os.sep): + workspace_fp += os.sep + logger.warning("Parser - parse_one_office - workspace_fp did not end with " + "trailing '/' as expected by parser") + + # set up workspace for parser + for z in range(0, 1): + + if os.path.exists(os.path.join(workspace_fp,str(z))): + shutil.rmtree(os.path.join(workspace_fp,str(z)), ignore_errors=True) + + if not os.path.exists(os.path.join(workspace_fp,str(z))): + os.mkdir(os.path.join(workspace_fp,str(z))) + os.chmod(os.path.join(workspace_fp, str(z)), 0o777) + + # * function declaration - add_one_office_opt_full * + + # char * input_account_name + # char * input_library_name + # char * input_fp + # char * input_fn + # char * workspace_fp + # char * image_fp + # char * write_to_filename + # int * unique_doc_num + # int * user_blok_size + # int strip_header + # int table_extract + # int smart_chunking + # int max_chunk_size + # int encoding_style + # int get_header_text + # int table_grid + # int save_images + # int logger_level + # char * debug_log_file + # int input_debug_mode + + # if any issue loading module, will be captured at .get_module_office_parser() + _mod = Utilities().get_module_office_parser() + + main_handler = _mod.add_one_office_opt_full + + main_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_char_p, c_char_p, c_int, c_int, c_int, c_int, c_int, + c_int, c_int, c_int, c_int, c_int, c_int, c_char_p, c_int) + + main_handler.restype = c_int + + # pull target block size from library parameters + user_block_size_c = c_int(self.chunk_size) + + if not self.account_name: + self.account_name = "llmware" + + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + if not fp.endswith(os.sep): + fp += os.sep + + fp_c = create_string_buffer(fp.encode('ascii', 'ignore')) + fn_c = create_string_buffer(fn.encode('ascii', 'ignore')) + + workspace_fp_c = create_string_buffer(workspace_fp.encode('ascii', 'ignore')) + + image_fp = self.parser_image_folder + + if not image_fp.endswith(os.sep): + image_fp += os.sep + logger.warning("Parser - parse_one_office - adding '/' to image_fp as expected by c parser") + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + write_to_filename = "office_internal_test0.txt" + write_to_fn_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + # defaults to 0 + if self.strip_header: + strip_header = c_int(1) + else: + strip_header = c_int(0) + + if self.get_tables: + table_extract = c_int(1) + else: + table_extract = c_int(0) + + smart_chunking = c_int(self.smart_chunking) + + # by default - 1 = get header text || turn off = 0 + if self.get_header_text: + get_header_text = c_int(1) + else: + get_header_text = c_int(0) + + if self.table_grid: + table_grid = c_int(1) + else: + table_grid = c_int(0) + + if self.encoding == "ascii": + encoding_style = c_int(0) + elif self.encoding == "utf-8": + encoding_style = c_int(2) + else: + encoding_style = c_int(2) + + max_chunk_size = c_int(self.max_chunk_size) + + if self.get_images: + save_images = c_int(1) # TRUE - get images + else: + save_images = c_int(0) # FALSE - no images + + logger.debug("Parser - parse_one_office - start parsing of office document...") + + # placeholder for now - not used + unique_doc_num_c = c_int(34) + + if self.use_logging_file: + input_debug_mode = c_int(60) + else: + input_debug_mode = c_int(0) + + if self.logger_level <= 10: + logger_level = c_int(self.logger_level) + else: + # unless in debug mode, suppress informational updates from parsers + logger_level = c_int(40) + + dlf_fp = os.path.join(self.parser_folder, self.parser_log_name) + debug_log_file = create_string_buffer(dlf_fp.encode('ascii', 'ignore')) + + pages_created = main_handler(account_name, library_name, fp_c, fn_c, workspace_fp_c, image_fp_c, + write_to_fn_c, unique_doc_num_c, user_block_size_c, + strip_header, table_extract, smart_chunking, max_chunk_size, + encoding_style, get_header_text, table_grid, save_images, + logger_level, debug_log_file, input_debug_mode) + + # self.library.image_path + output = self.convert_parsing_txt_file_to_json(file_path=self.parser_tmp_folder,fn=write_to_filename) + + if len(output) > 0: + self.parser_output += output + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_one_pdf (self, fp, fn, save_history=True): + + """ Parse one pdf document at selected file path and file name. """ + + # 0.3.2 - adding expanded configs with text chunking and logging options + + # check that path exists + if not os.path.exists(os.path.join(fp,fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(fp, fn)}") + + system = platform.system().lower() + + if system == "linux": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == 'aarch64': + # re-initiating support for linux aarch64 + return self.parse_one_pdf_deprecated_031(fp,fn,save_history=save_history) + + if system == "darwin": + + try: + machine = os.uname().machine.lower() + except: + machine = "na" + + if machine == "x86_64": + + error_msg = ("Mac x86 detected as OS - this is not a supported platform. Support " + "was deprecated in llmware version 0.2.6 and removed in llmware version 0.3.9. " + "Options - move to Mac Metal (M1+), back-level llmware to supported version, or " + "if urgent requirement for Mac x86, please raise ticket on github.") + + raise LLMWareException(message=error_msg) + + # end - deprecation routing + + # if any issue loading module, will be captured at .get_module_pdf_parser() + _mod_pdf = Utilities().get_module_pdf_parser() + + pdf_handler = _mod_pdf.add_one_pdf_opts + + # * function declaration - add_one_pdf_opts * + + # char * account_name + # char * library_name + # char * input_fp + # char * input_filename + # char * input_images_fp + # char * write_to_filename + # int user_blok_size + # int unique_doc_num + # int strip_header + # int table_extract + # int smart_chunking + # int max_chunk_size + # int encoding_style + # int get_header_text + # int table_grid + # int save_images + # int logger_level + # char * debug_log_file + # int input_debug_mode + + pdf_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_int, + c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int, c_int, + c_char_p, c_int) + + pdf_handler.restypes = c_int + + # prepare all of the inputs to invoke the c library + + t0 = time.time() + + # config options pulled from the Library object + if not self.account_name: + acct_name = "llmware" + + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + # fp = passed as parameter -> this is the input file path folder containing the .PDF docs to be parsed + + if not fp.endswith(os.sep): + fp += os.sep + + fp_c = create_string_buffer(fp.encode('ascii', 'ignore')) + + fn_c = create_string_buffer(fn.encode('ascii', 'ignore')) + + image_fp = self.parser_tmp_folder + if not image_fp.endswith(os.sep): + image_fp += os.sep + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + # prep parameters passed in the method invocation above + write_to_filename = "pdf_internal_test0.txt" + write_to_filename_c = create_string_buffer(write_to_filename.encode('ascii','ignore')) + + # pull target block size from library parameters + user_block_size = c_int(self.chunk_size) + + # defaults to 0 + if self.strip_header: + strip_header = c_int(1) + else: + strip_header = c_int(0) + + # update - expose configs for table extraction strategy + if self.get_tables: + if 0 <= self.table_strategy <= 2: + table_extract = c_int(self.table_strategy) + else: + table_extract = c_int(1) + else: + table_extract = c_int(0) + + smart_chunking = c_int(self.smart_chunking) + + # by default - 1 = get header text || turn off = 0 + if self.get_header_text: + get_header_text = c_int(1) + else: + get_header_text = c_int(0) + + if self.table_grid: + table_grid = c_int(1) + else: + table_grid = c_int(0) + + if self.encoding == "ascii": + encoding_style = c_int(0) + elif self.encoding == "utf-8": + encoding_style = c_int(2) + else: + encoding_style = c_int(2) + + max_chunk_size = c_int(self.max_chunk_size) + + if self.get_images: + save_images = c_int(1) # TRUE - get images + else: + save_images = c_int(0) # FALSE - no images + + unique_doc_num_c = c_int(34) + + if self.use_logging_file: + input_debug_mode = c_int(60) + else: + input_debug_mode = c_int(0) + + if self.logger_level <= 10: + logger_level = c_int(self.logger_level) + else: + # unless in debug mode, then suppress information updates from parsers + logger_level = c_int(40) + + dlf_fp = os.path.join(self.parser_folder, self.parser_log_name) + debug_log_file = create_string_buffer(dlf_fp.encode('ascii', 'ignore')) + + logger.debug("Parser - parse_one_pdf - starting pdf_parser ...") + + # main call into pdf parser + + pages_created = pdf_handler(account_name, library_name, fp_c, fn_c, image_fp_c, + write_to_filename_c, user_block_size, + unique_doc_num_c, strip_header, table_extract, smart_chunking, + max_chunk_size, encoding_style, get_header_text, table_grid, + save_images, logger_level, debug_log_file, input_debug_mode) + + logger.debug(f"Parser - parse_one_pdf - completed pdf_parser - time taken: {time.time()-t0}") + + output = self.convert_parsing_txt_file_to_json(file_path=self.parser_tmp_folder,fn=write_to_filename) + + if len(output) > 0: + self.parser_output += output + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_one_pdf_by_ocr_images(self, input_fp, input_fn, save_history=True): + + """ Parse one 'scanned' pdf document at selected file path and file name. """ + + # check that path exists + if not os.path.exists(os.path.join(input_fp, input_fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(input_fp, input_fn)}") + + # Designed for parse of a single PDF_BY_OCR - no storage, no link into Library + # --output returned as in-memory list of Dicts + + # set counters + output = [] + doc_id = 0 + + ext = input_fn.split(".")[-1] + + if ext == "pdf": + + doc_fn = Utilities().secure_filename(input_fn) + + output_by_page = ImageParser(self).process_pdf_by_ocr(input_fp, input_fn) + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + counter = 0 + for i, pages in enumerate(output_by_page): + for j, blocks in enumerate(pages): + + new_entry = ("text", "pdf-ocr", (j+1, 0), counter, "", "", doc_fn, "", blocks, "", + "", blocks, blocks, "", blocks, "", "", "", "", "") + + # creates a single 'unbound' parsing output dict -> no storage + parsing_output_dict = self.create_one_parsing_output_dict(counter, + new_entry, meta, coords_dict, + dialog_value="false") + + output.append(parsing_output_dict) + self.parser_output.append(parsing_output_dict) + + counter += 1 + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_one_image(self, input_fp, input_fn, save_history=True): + + """ Parse one image document at selected file path and file name. """ + + # Designed to parse a single image using OCR - no storage or link to library + + # check that path exists + if not os.path.exists(os.path.join(input_fp, input_fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(input_fp, input_fn)}") + + # set counters + output= [] + counter = 0 + ext = input_fn.split(".")[-1].lower() + + if ext in self.ocr_types: + + doc_fn = Utilities().secure_filename(input_fn) + ocr_output = ImageParser(self).process_ocr(input_fp, input_fn) + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + for j, blocks in enumerate(ocr_output): + + new_entry = ("text", "pdf-ocr", (1, 0), counter, "", "", doc_fn, "", blocks, "", + "", blocks, blocks, "", blocks, "", "", "", "", "") + + # creates a single 'unbound' parsing output dict -> no storage + parsing_output_dict = self.create_one_parsing_output_dict(counter, new_entry, meta, coords_dict, + dialog_value="false") + + output.append(parsing_output_dict) + self.parser_output.append(parsing_output_dict) + + counter += 1 + + if save_history: + ParserState().save_parser_output(self.parser_job_id, output) + + return output + + def parse_one_text(self, input_fp, input_fn, save_history=True): + + """ Parse one text-based document at selected file path and file name. """ + + # check that path exists + if not os.path.exists(os.path.join(input_fp, input_fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(input_fp, input_fn)}") + + # set counters + output = [] + content_type = "text" + parser_output = [] + counter = 0 + + file_type = input_fn.split(".")[-1].lower() + + if file_type not in self.text_types: + return output + + # sub-routing by type of text file to appropriate handler + + if file_type in ["txt", "md"]: + # will parse as text + parser_output = TextParser(self).text_file_handler (input_fp, input_fn) + content_type = "text" + file_type = "txt" + + if file_type.lower() in ["csv", "tsv"]: + # will parse as table + interpret_as_table=True + + if file_type.lower() == "tsv": + separator="\t" + else: + separator="," + + parser_output = TextParser(self).csv_file_handler(input_fp, input_fn, interpret_as_table=True, + delimiter=separator) + + content_type = "text" + # file_type = "csv" + file_type = file_type.lower() + if interpret_as_table: + content_type = "table" + + if file_type.lower() in ["json","jsonl"]: + # will parse each line item as separate entry + + interpret_as_table=False + keys = ["text"] + parser_output = TextParser(self).jsonl_file_handler(input_fp,input_fn, + key_list=keys, + interpret_as_table=interpret_as_table, + separator="\n") + content_type = "text" + file_type = "jsonl" + if interpret_as_table: + content_type = "table" + + # consolidate output + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + for j, blocks in enumerate(parser_output): + + if content_type == "text": + new_entry = ("text", file_type, (1, 0), counter, "", "", input_fn, "", blocks, "", + "", blocks, blocks, "", blocks, "", "", "", "", "") + else: + # could be table if csv file -> in this case, keep both text [11] and table [7] + new_entry = ("table", file_type, (1, 0), counter, "", "", input_fn, blocks, blocks, "", + "", blocks, blocks, "", blocks, "", "", "", "", "") + + # creates a single 'unbound' parsing output dict -> no storage + parsing_output_dict = self.create_one_parsing_output_dict(counter, + new_entry, meta, coords_dict, + dialog_value="false") + + output.append(parsing_output_dict) + self.parser_output.append(parsing_output_dict) + + counter += 1 + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_one_dialog(self, input_fp, input_fn, save_history=True): + + """ Parse one dialog transcript document at selected file path and file name. """ + + # Designed as single dialog parse - no storage or link to library + # --note: only supports AWS dialog standard for now + + # check that path exists + if not os.path.exists(os.path.join(input_fp, input_fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(input_fp, input_fn)}") + + # set counters + counter = 0 + output = [] + + ext = input_fn.split(".")[-1].lower() + + if ext == "json": + + dp_output = DialogParser(self).parse_aws_json_file_format(input_fp, input_fn) + + for i, blocks in enumerate(dp_output): + + # iterate thru each block -> add to metadata + speaker_name = blocks["speaker_name"] + + meta = {"author": speaker_name, "modified_date": "", "created_date": "", "creator_tool": ""} + + coords_dict = {"coords_x": blocks["start_time"], + "coords_y": blocks["stop_time"], + "coords_cx": 0, + "coords_cy": 0} + + text_entry = blocks["text"] + + # conforming file format with full path of dialog intake path + + format_type = "aws_json" + + new_entry = ("text", format_type, (1, 0), counter, "", "", input_fn, + text_entry, text_entry, "", "", text_entry, text_entry, "", text_entry, + "", "", "", "", "") + + # creates a single 'unbound' parsing output dict -> no storage + parsing_output_dict = self.create_one_parsing_output_dict(counter, + new_entry, meta, coords_dict, + dialog_value="true") + + output.append(parsing_output_dict) + self.parser_output.append(parsing_output_dict) + counter += 1 + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_one_voice(self, input_fp, input_fn, save_history=True, + chunk_by_segment=True, remove_segment_markers=True, real_time_progress=True): + + """ Parse one WAV document at selected file path and file name. """ + + # Designed to parse a single WAV/voice file - no storage or linkage to library + + # check that path exists + if not os.path.exists(os.path.join(input_fp, input_fn)): + raise LLMWareException(message=f"Parser - could not find path - " + f"{os.path.join(input_fp, input_fn)}") + + # set counters + counter = 0 + output = [] + + ext = input_fn.split(".")[-1].lower() + + if ext in self.voice_types: + + parser_output = VoiceParser(self, + chunk_size=self.chunk_size, + max_chunk_size=self.max_chunk_size, + chunk_by_segment=chunk_by_segment, + remove_segment_markers=remove_segment_markers, + real_time_progress=real_time_progress).add_voice_file(input_fp, input_fn) + + if not chunk_by_segment: + text_chunks_only = [] + for chunks in parser_output: + text_chunks_only.append(chunks["text"]) + + new_output, new_blocks, new_pages = self._write_output_to_dict(text_chunks_only, input_fn, + content_type="text", file_type="voice-wav") + + self.parser_output.append(new_output) + output += new_output + + else: + + for i, blocks in enumerate(parser_output): + + # iterate thru each block -> add to metadata + speaker_name = blocks["speaker"] + + meta = {"author": speaker_name, "modified_date": "", "created_date": "", "creator_tool": ""} + + coords_dict = {"coords_x": blocks["start_time"], "coords_y": blocks["end_time"], + "coords_cx": blocks["start_segment"], "coords_cy": blocks["end_segment"]} + + text_entry = blocks["text"] + + # conforming file format with full path of dialog intake path + + format_type = "voice-wav" + + new_entry = ("text", format_type, (1, 0), i, "", "", input_fn, + "", text_entry, "", "", text_entry, text_entry, "", text_entry, + "", "", "", "", "") + + entry_output = self.create_one_parsing_output_dict(i, new_entry, meta, coords_dict, + dialog_value="false") + + self.parser_output.append(entry_output) + + # return value is output + output.append(entry_output) + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def query_parser_state(self, query, results=None, remove_stop_words=True): + + """ Runs an in-memory 'fast search' against a set of parsed output json dictionaries. """ + + if not results: + results = self.parser_output + + output = Utilities().fast_search_dicts(query,results, text_key="text",remove_stop_words=remove_stop_words) + + return output + + def input_build_folder(self, fp_list, exclude_if_already_in_library=True): + + """ Creates a single 'input_build_folder' by consolidating the files across multiple folders, provided in + input list of file paths. It will accept only one copy of a particular file, based on the + first version passed in the fp_list. If exclude_if_already_in_library == True (default), then + any files already in the library will also be excluded. """ + + # once the input_build_folder is created, it can be passed to any parsing method + + library_docs = None + + if not self.library: + exclude_if_already_in_library = False + + if exclude_if_already_in_library: + + # will get a list of all of the distinct files already in the library + library_docs = CollectionRetrieval(self.library_name, + account_name=self.account_name).get_distinct_list("file_source") + + # create tmp workspace for new_input_folder + new_input_folder = os.path.join(self.parser_folder, "input_build" + os.sep) + + # if new_input_folder already created, then delete + if os.path.exists(new_input_folder): + shutil.rmtree(new_input_folder) + + # create and start fresh + if not os.path.exists(new_input_folder): + os.mkdir(new_input_folder) + os.chmod(new_input_folder, 0o777) + + deduped_list = [] + dupe_files = [] + lib_match_list = [] + + for folder in fp_list: + + input_files = os.listdir(folder) + + for file in input_files: + if file not in deduped_list: + dupe = 0 + if exclude_if_already_in_library: + for lib_file in library_docs: + if os.sep in lib_file: + lib_file = lib_file.split(os.sep)[-1] + if file == lib_file: + dupe = 1 + lib_match_list.append(file) + break + if dupe == 0: + deduped_list.append(file) + shutil.copy(os.path.join(folder,file), os.path.join(new_input_folder,file)) + else: + # copies the full path of the file that is being excluded + dupe_files.append(os.path.join(folder, file)) + + output_info = {"new_input_folder": new_input_folder, + "file_count": len(deduped_list), + "files_included": deduped_list, + "duplicates_removed":dupe_files, + "files_in_library_already": lib_match_list} + + return output_info + + def delete_input_build_folder(self): + + """ Deletes an input build folder - at end of parsing transaction(s) using input_builder_folder """ + + input_build_folder = os.path.join(self.parser_folder, "input_build" + os.sep) + + # if new_input_folder already created, then delete + if os.path.exists(input_build_folder): + shutil.rmtree(input_build_folder) + + return True + + def duplicate_file_already_in_library(self, fp): + + """ Reviews the files in input folder path, and checks if any of those files have blocks of information + in the library database collection. """ + + existing_docs_in_collection = CollectionRetrieval(self.library_name, + account_name=self.account_name).get_distinct_list("file_source") + + input_files = os.listdir(fp) + + no_dupes_list = [] + matching_file_names = [] + + for file in input_files: + + match_found = 0 + + for existing_file in existing_docs_in_collection: + + if os.sep in existing_file: + # split to get base file name + existing_file = existing_file.split(os.sep)[-1] + + if file == existing_file: + matching_file_names.append(file) + match_found = 1 + break + + if match_found == 0: + no_dupes_list.append(file) + + duplicate_check = {"not_in_library": no_dupes_list, "in_library": matching_file_names} + + return duplicate_check + + def basic_library_duplicate_check(self, fn): + + """ Checks if file is already part of the copied upload files for the library, and returns + True if file is found, and False if not found, e.g., 'new' to the library """ + + in_library = False + + # run comparison with existing files in library copy path + if self.library: + if os.path.exists(self.library.file_copy_path): + existing_files = os.listdir(self.library.file_copy_path) + + if fn in existing_files: + in_library = True + + return in_library + + def parse_csv_config(self,fp, fn, cols=None, mapping_dict=None, delimiter=","): + + """ Designed for intake of a 'pseudo-db csv table' and will add rows to library with mapped keys. + + Inputs: + -- csv folder path + csv file name + -- cols = # of expected column entries in each row of the CSV + -- mapping dict = assigns key names to columns, starting with 0 for first column + e.g., {"text": 4, "doc_ID": 2, "key1": 3} + + Requirements: + -- must have a "text" key in the mapping dictionary + -- optional doc_ID and block_ID - if found, will over-write the normal library indexes + -- all other keys will be saved as 'metadata' and added to the library block row in "special_field1" + + Note: this feature is currently only supported for Mongo - SQL DB support will follow. + """ + + # method requires library loaded in the Parser + + if not self.library: + raise LLMWareException(message="Parsing of a configured CSV file requires a library object to " + "be connected to the parser state.") + + # if found in mapping dict, then will over-write + reserved_keys = ["text", "doc_ID", "block_ID"] + + rejected_rows = [] + + if not mapping_dict: + raise LLMWareException(message="Parsing of a configured CSV file requires a mapping dictionary so that " + "the table attributes can be properly mapped.") + + if not cols: + raise LLMWareException(message="Parsing of a configured CSV file requires a defined column structure and " + "a specified number of columns to ensure accurate mapping.") + + # file type + ft = fn.split(".")[-1].lower() + if ft == "tsv": + delimiter="\t" + + # will iterate through csv file + input_csv = os.path.join(fp, fn) + + output = Utilities.file_load(input_csv,delimiter=delimiter,encoding='utf-8-sig',errors='ignore') + + added_row_count = 0 + total_row_count = 0 + added_doc_count = 0 + + for i, rows in enumerate(output): + + text = "" + doc_id = None + block_id = None + metadata = {} + + if len(rows) != cols: + bad_entry = {"index": i, "row": rows} + rejected_rows.append(bad_entry) + + else: + # confirmed that row has the correct number of entries + + for keys, values in mapping_dict.items(): + + if keys == "text": + if mapping_dict["text"] < len(rows): + text = rows[mapping_dict["text"]] + + if keys == "doc_ID": + if mapping_dict["doc_ID"] < len(rows): + doc_id = rows[mapping_dict["doc_ID"]] + + if keys == "block_ID": + if mapping_dict["block_ID"] < len(rows): + block_id = rows[mapping_dict["block_ID"]] + + if keys not in reserved_keys: + if values < len(rows): + metadata.update({keys:rows[values]}) + + if text.strip(): + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + # note: if using SQL-based DB, then will save the metadata as a text string + if LLMWareConfig().get_config("collection_db") in ["sqlite", "postgres"]: + metadata = str(metadata) + + new_row_entry = ("text", "custom_csv", (1, 0), total_row_count, "", "", fn, + text, text, "", "", text, text, "", text, "", "", metadata, "", "") + + # set attributes custom + if doc_id: + try: + self.library.doc_ID = int(doc_id) + added_doc_count += 1 + except: + logger.debug(f"Parser - parse_csv_config - doc_ID expected to be integer - " + f"can not apply custom doc ID - {doc_id} - will use default " + f"library document increment") + + if block_id: + self.library.block_ID = block_id + else: + self.library.block_ID += 1 + + # write row to database + + entry_output = self.add_create_new_record(self.library, + new_row_entry, + meta, + coords_dict, + dialog_value="false") + added_row_count += 1 + + total_row_count += 1 + + # update overall library counter at end of parsing + + if len(output) > 0: + + if added_doc_count == 0: + added_doc_count += 1 + + dummy = self.library.set_incremental_docs_blocks_images(added_docs=added_doc_count, + added_blocks=added_row_count, + added_images=0, added_pages=0) + + output = {"rows_added": added_row_count, "rejected_count": len(rejected_rows), "rejected_rows": rejected_rows} + + return output + + def parse_json_config(self,fp, fn, mapping_dict=None): + + """ Designed for intake of a 'pseudo-db json/jsonl table' and will add rows to library with mapped keys. + + Inputs: + -- json folder path + json file name + -- cols = # of expected column entries in each row of the CSV + -- mapping dict = assigns llmware library key names to keys in the json + e.g., {"text": "output", "doc_ID": "ID", "key1": "special_field1", "key2": "special_field2", "key3":"special_field3"} + + Requirements: + -- must have a "text" key in the mapping dictionary + -- optional doc_ID and block_ID - if found, will over-write the normal library indexes + -- all other keys will be saved as 'metadata' and added to the library block row in "special_field1" + + Note: this feature is currently only supported for Mongo - SQL DB support will follow. + """ + + # method requires a library loaded in the Parser + + if not self.library: + raise LLMWareException(message="Parsing of a configured CSV file requires a library object to " + "be connected to the parser state.") + + # if found in mapping dict, then will over-write + reserved_keys = ["text", "doc_ID", "block_ID"] + + rejected_rows = [] + + if not mapping_dict: + raise LLMWareException(message="Parsing of a configured JSON/JSONL file requires a mapping dictionary so that " + "the table attributes can be properly mapped.") + + # will iterate through json/jsonl file + ft = fn.split(".")[-1].lower() + + if ft not in ["json", "jsonl"]: + raise LLMWareException(message=f"File type not recognized as JSON/JSONL - {ft}") + + output = [] + + if ft == "json": + output = json.load(open(os.path.join(fp, fn), "r")) + + if ft == "jsonl": + my_file = open(os.path.join(fp, fn), 'r', encoding='utf-8-sig',errors='ignore') + + output = [] + for i, lines in enumerate(my_file): + row_tmp = json.loads(lines) + output.append(row_tmp) + + my_file.close() + + added_row_count = 0 + total_row_count = 0 + added_doc_count = 0 + + for i, rows in enumerate(output): + + text = "" + doc_id = None + block_id = None + metadata = {} + + for keys, values in mapping_dict.items(): + + if keys == "text": + if values in rows: + text = rows[values] + + if keys == "doc_ID": + if values in rows: + doc_id = rows[values] + + if keys == "block_ID": + if values in rows: + block_id = rows[values] + + if keys not in reserved_keys: + metadata.update({keys:rows[values]}) + + if text.strip(): + + meta = {"author": "", "modified_date": "", "created_date": "", "creator_tool": ""} + coords_dict = {"coords_x": 0, "coords_y": 0, "coords_cx": 0, "coords_cy": 0} + + # conforming file format with full path of dialog intake path + metadata = str(metadata) + + new_row_entry = ("text", "custom_json", (1, 0), total_row_count, "", "", fn, + text, text, "", "", text, text, "", text, "", "", metadata, "", "") + + # set attributes custom + if doc_id: + try: + self.library.doc_ID = int(doc_id) + added_doc_count += 1 + except: + logger.debug(f"Parser - parse_json_config - doc_ID expected to be integer - " + f"can not apply custom doc ID - {doc_id} - will use default library " + f"document increment") + + if block_id: + self.library.block_ID = block_id + else: + self.library.block_ID += 1 + + # write row to database + + entry_output = self.add_create_new_record(self.library, + new_row_entry, + meta, + coords_dict, + dialog_value="false") + added_row_count += 1 + + total_row_count += 1 + + # update overall library counter at end of parsing + + if len(output) > 0: + + if added_doc_count == 0: + added_doc_count += 1 + + dummy = self.library.set_incremental_docs_blocks_images(added_docs=added_doc_count, + added_blocks=added_row_count, + added_images=0, added_pages=0) + + output = {"rows_added": added_row_count, "rejected_count": len(rejected_rows), "rejected_rows": rejected_rows} + + return output + + def ocr_images_in_library(self, add_to_library=False, chunk_size=400, min_size=10, + realtime_progress=True): + + """ Assumes that a Library is passed in the Parser constructor, and that the Library already contains + some parsed content with at least some images found. This method will identify the images extracted + across the entire library, and then run an OCR against each image looking for text to extract, apply + text chunking rules, and then save the new OCR-extracted text in the library database. + + Output, by default, is verbose and displays real-time progress from the OCR to be able to evaluate the + quality before confirming `add_to_library = True`. To remove the verbose screen output, set + `realtime_progress = False`. """ + + if not self.library: + raise LLMWareException(message="Exception: Parser - ocr_images_in_library - is intended to be used " + "in conjunction with a loaded Library. To use, you can call the " + "Library class convenience method - .ocr_on_images method.") + + from importlib import util + if not util.find_spec("pytesseract"): + raise LLMWareException(message="Exception: Parser - ocr_images_in_library - requires additional " + "dependencies to be installed on your system. \nTo use this method, " + "please implement two prerequisites:" + "\n1. pytesseract - pip3 install pytesseract" + "\n2. lib tesseract - core library and can be installed:" + "\n -mac os: brew install tesseract" + "\n -ubuntu: sudo apt install libtesseract-dev" + "\n -windows: use GUI download installer") + + library_name = self.library.library_name + image_path = self.library.image_path + + # check here to see the images extracted from the original parsing + if realtime_progress: + logger.info(f"update: image source file path: {image_path}") + + # query the collection DB by content_type == "image" + image_blocks = CollectionRetrieval(library_name).filter_by_key("content_type", "image") + doc_update_list = {} + new_text_created = 0 + + # iterate through the image blocks found + for i, block in enumerate(image_blocks): + + # "external_files" points to the image name that will be found in the image_path above for the library + img_name = block["external_files"] + + # each doc_ID is unique for the library collection + doc_id = block["doc_ID"] + + # block_IDs are unique only for the document, and generally run in sequential ascending order + block_id = block["block_ID"] + + # note: _id not used, but it is a good lookup key that can be easily inserted in special_field1 below + bid = block["_id"] + + # preserve_spacing == True will keep \n \r \t and other white space + # preserve_spacing == False collapses the white space into a single space for 'more dense' text only + output = ImageParser(text_chunk_size=chunk_size).process_ocr(image_path, img_name, preserve_spacing=False) + + if realtime_progress: + logger.info(f"Parser - ocr_images_in_library - realtime progress- ocr output: {output}") + + # good to do a test run with 'add_to_library' == False before writing to the collection + if add_to_library: + + for text_chunk in output: + + if text_chunk.strip(): + + # optional to keep only more substantial chunks of text + if len(text_chunk) > min_size: + + # ad hoc tracker to keep incrementing the block_id for every new image in a particular doc + if doc_id in doc_update_list: + new_block_id = doc_update_list[doc_id] + doc_update_list.update({doc_id: new_block_id + 1}) + else: + new_block_id = 100000 + doc_update_list.update({doc_id: new_block_id + 1}) + + new_block = block + + # feel free to adapt these attributes to fit for purpose + new_block.update({"block_ID": new_block_id}) + new_block.update({"content_type": "text"}) + new_block.update({"embedding_flags": {}}) + new_block.update({"text_search": text_chunk}) + + # writes a special entry in 'special_field1' of the database + # this special entry captures the link back to the original 'image' block + # it can be unpacked by splitting on '&' and '-' to retrieve the doc_id and block_id + + output = f"document-{doc_id}&block-{block_id}" + new_block.update({"special_field1": output}) + + # new _id will be assigned by the database directly + if "_id" in new_block: + del new_block["_id"] + + if realtime_progress: + logger.info(f"Parser - ocr_images_in_library - new text block - {new_text_created} - " + f"{doc_id} - {block_id} - {text_chunk} - {new_block}") + + # creates the new record + CollectionWriter(library_name).write_new_parsing_record(new_block) + + new_text_created += 1 + + return new_text_created + + def parse_pdf_deprecated_026(self, fp, write_to_db=True, save_history=True, image_save=1): + + """ Main PDF parser method through version 0.2.6 - deprecated - wraps ctypes interface to call PDF parser. + Will be removed in future release. """ + + output = [] + + write_to_filename = "pdf_parse_output_0.txt" + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + unique_doc_num = -1 + else: + write_to_db_on = 0 + unique_doc_num = int(self.file_counter) + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("warning: Parser().parse_pdf - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.error(f"warning: Parser().parse_pdf - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place " + f"the file in /parser_history path.") + + # * function declaration for .add_pdf_main_llmware * + # char * input_account_name + # char * input_library_name + # char * input_fp + # char * db + # char * db_uri_string + # char * db_name + # char * db_user_name + # char * db_pw + # char * input_images_fp + # int input_debug_mode + # int input_image_save_mode + # int write_to_db_on + # char * write_to_filename + # int user_blok_size + # int unique_doc_num + # int status_manager_on + # int status_manager_increment + # char * status_job_id + + # if any issue loading module, will be captured at .get_module_pdf_parser() + _mod_pdf = Utilities().get_module_pdf_parser() + + # pdf_handler = _mod_pdf.add_pdf_main_customize_parallel + pdf_handler = _mod_pdf.add_pdf_main_llmware_config + + pdf_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_char_p, c_int, c_int, c_int, c_char_p, c_int, c_int, c_int, c_int, c_char_p) + + pdf_handler.restypes = c_int + + # prepare all of the inputs to invoke the c library + + t0 = time.time() + + # config options pulled from the Library object + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + # image_fp = self.library.image_path + image_fp = self.parser_image_folder + + if not image_fp.endswith(os.sep): + image_fp += os.sep + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + input_collection_db_path = LLMWareConfig().get_db_uri_string() + + collection_db_path_c = create_string_buffer(input_collection_db_path.encode('ascii', 'ignore')) + + # fp = passed as parameter -> this is the input file path folder containing the .PDF docs to be parsed + if not fp.endswith(os.sep): + fp += os.sep + + fp_c = create_string_buffer(fp.encode('ascii', 'ignore')) + + # debug_mode global parameter + # "on" = 1 + # "file name only" = 2 + # "deep debug" = 3 + # "off" = 0 & all other values + + # pull debug mode 'verbosity' levels from LLMWareConfig + debug_mode = LLMWareConfig.get_config("debug_mode") + + supported_options = [0, 1, 2, 3] + + if debug_mode not in supported_options: + debug_mode = 0 + + input_debug_mode = c_int(debug_mode) # default - 0 = "off" + input_image_save_mode = c_int(image_save) # default - 1 = "on" | use 0 = "off" in production + + write_to_db_on_c = c_int(write_to_db_on) + write_to_filename_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + # pull target block size from library parameters + user_block_size = c_int(self.block_size_target_characters) # standard 400-600 + + # unique_doc_num -> if <0: interpret as "OFF" ... if >=0 then use and increment doc_id directly + # unique_doc_num = -1 + unique_doc_num_c = c_int(unique_doc_num) + + # db credentials + db_user_name = self.collection_db_username + db_user_name_c = create_string_buffer(db_user_name.encode('ascii', 'ignore')) + + db_pw = self.collection_db_password + db_pw_c = create_string_buffer(db_pw.encode('ascii', 'ignore')) + + db = LLMWareConfig.get_config("collection_db") + + db = create_string_buffer(db.encode('ascii', 'ignore')) + db_name = account_name + + status_manager_on = c_int(1) + status_manager_increment = c_int(10) + status_job_id = create_string_buffer("1".encode('ascii', 'ignore')) + + # + # * main call to pdf library * + # + + logger.info("Parser - start parsing of PDF Documents...") + + pages_created = pdf_handler(account_name, library_name, fp_c, db, collection_db_path_c, db_name, + db_user_name_c, db_pw_c, + image_fp_c, + input_debug_mode, input_image_save_mode, write_to_db_on_c, + write_to_filename_c, user_block_size, unique_doc_num_c, + status_manager_on, status_manager_increment, status_job_id) + + logger.info(f"Parser - completed parsing of pdf documents - time taken: {time.time() - t0}") + + if write_to_db_on == 0: + # package up results in Parser State + parser_output = self.convert_parsing_txt_file_to_json(self.parser_image_folder, write_to_filename) + if len(parser_output) > 0: + last_entry = parser_output[-1] + last_doc_id = last_entry["doc_ID"] + + self.file_counter = int(last_doc_id) + + logger.info(f"Parser - adding new entries to parser output state - {len(parser_output)}") + + self.parser_output += parser_output + output += parser_output + + if save_history: + ParserState().save_parser_output(self.parser_job_id, parser_output) + + return output + + def parse_office_deprecated_027(self, input_fp, write_to_db=True, save_history=True): + + """ Deprecated - primary method interface into Office parser with more db configuration options - + implemented starting with llmware-0.2.0 and deprecated as of 0.2.7 - will be removed in future + releases. """ + + output = [] + + # used internally by parser to capture text + write_to_filename = "office_parser_output_0.txt" + + # must have three conditions in place - (a) user selects, (b) ping successfully, and (c) library loaded + if write_to_db and self.parse_to_db and self.library: + write_to_db_on = 1 + unique_doc_num = -1 + else: + write_to_db_on = 0 + unique_doc_num = int(self.file_counter) + + # warning to user that no library loaded in Parser constructor + if write_to_db and not self.library: + logger.warning("Parser - parse_office - request to write to database but no library loaded " + "in Parser constructor. Will write parsing output to file and will place the " + "file in Parser /parser_history path.") + + # warning to user that database connection not found + if write_to_db and not self.parse_to_db: + logger.warning(f"Parser - parse_office - could not connect to database at " + f"{self.collection_path}. Will write parsing output to file and will place the " + f"file in Library /images path.") + + # designed for bulk upload of office parse into library structure + + if not input_fp.endswith(os.sep): + input_fp += os.sep + + office_fp = input_fp + + workspace_fp = os.path.join(self.parser_tmp_folder, "office_tmp" + os.sep) + + if not os.path.exists(workspace_fp): + os.mkdir(workspace_fp) + os.chmod(workspace_fp, 0o777) + + # start timing track for parsing job + t0 = time.time() + + # only one tmp work folder used currently - can consolidate over time + for z in range(0, 5): + + if os.path.exists(os.path.join(workspace_fp, str(z))): + shutil.rmtree(os.path.join(workspace_fp, str(z)), ignore_errors=True) + + if not os.path.exists(os.path.join(workspace_fp, str(z))): + os.mkdir(os.path.join(workspace_fp, str(z))) + os.chmod(os.path.join(workspace_fp, str(z)), 0o777) + + # end -initialize workspace + + # if any issue loading module, will be captured at .get_module_office_parser() + _mod = Utilities().get_module_office_parser() + + # new endpoint for llmware + main_handler = _mod.add_files_main_llmware_opt + + # * function declaration for add_files_main_llmware_opt * + + # char * input_account_name + # char * input_library_name + # char * input_fp + # char * workspace_fp + # char * db + # char * db_uri_string + # char * db_name + # char * db_user_name + # char * db_pw + # char * image_fp + # int input_debug_mode + # int write_to_db_on + # char * write_to_filename + # int unique_doc_num + # int user_blok_size + # int status_manager_on + # int status_manager_increment + # char * status_job_id) + + main_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, + c_int, c_int, c_char_p, c_int, c_int, c_int, c_int, + c_char_p) + + main_handler.restype = c_int + + # three inputs - account_name // library_name // fp to web_dir - files to be processed + # prep each string: account_name = create_string_buffer(py_account_str.encode('ascii','ignore')) + + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + fp_c = create_string_buffer(office_fp.encode('ascii', 'ignore')) + workspace_fp_c = create_string_buffer(workspace_fp.encode('ascii', 'ignore')) + + # debug_mode global parameter + # "on" = 1 + # "file name only" = 2 + # "deep debug" = 3 + # "off" = 0 & all other values + + debug_mode = LLMWareConfig.get_config("debug_mode") + + supported_options = [0, 1, 2, 3] + + if debug_mode not in supported_options: + debug_mode = 0 + + debug_mode_c = c_int(debug_mode) + + # image_fp = self.library.image_path + + image_fp = self.parser_image_folder + if not image_fp.endswith(os.sep): + image_fp += os.sep + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + # get db uri string + input_collection_db_path = LLMWareConfig().get_db_uri_string() + collection_db_path_c = create_string_buffer(input_collection_db_path.encode('ascii', 'ignore')) + + write_to_db_on_c = c_int(write_to_db_on) + + write_to_fn_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + # unique_doc_num is key parameter - if <0: will pull from incremental db, if >=0, then will start at this value + # unique_doc_num = -1 + unique_doc_num_c = c_int(unique_doc_num) + + # pull target block size from library parameters + user_block_size_c = c_int(self.block_size_target_characters) # standard 400-600 + + # db credentials + db_user_name = self.collection_db_username + db_user_name_c = create_string_buffer(db_user_name.encode('ascii', 'ignore')) + + db_pw = self.collection_db_password + db_pw_c = create_string_buffer(db_pw.encode('ascii', 'ignore')) + + db = LLMWareConfig.get_config("collection_db") + + db = create_string_buffer(db.encode('ascii', 'ignore')) + db_name = account_name + + status_manager_on_c = c_int(1) + status_manager_increment_c = c_int(10) + status_job_id_c = create_string_buffer("1".encode('ascii', 'ignore')) + + logger.info("Parser - parse_office - start parsing of office documents...") + + pages_created = main_handler(account_name, library_name, fp_c, workspace_fp_c, + db, collection_db_path_c, db_name, db_user_name_c, db_pw_c, + image_fp_c, debug_mode_c, write_to_db_on_c, write_to_fn_c, unique_doc_num_c, + user_block_size_c, status_manager_on_c, status_manager_increment_c, + status_job_id_c) + + logger.info(f"Parser - completed parsing of office documents - time taken: {time.time() - t0}") + + if write_to_db_on == 0: + # package up results in Parser State + parser_output = self.convert_parsing_txt_file_to_json(self.parser_image_folder, write_to_filename) + if len(parser_output) > 0: + last_entry = parser_output[-1] + last_doc_id = last_entry["doc_ID"] + + self.file_counter = int(last_doc_id) + + self.parser_output += parser_output + output += parser_output + + if save_history: + # save parser state + ParserState().save_parser_output(self.parser_job_id, parser_output) + + return output + + def parse_one_pdf_deprecated_031(self, fp, fn, save_history=True): + + """ Deprecated as of 0.3.2 - parse one pdf document at selected file path and file name - provides + fewer configuration options for text chunking and logging. """ + + # check that path exists + if not os.path.exists(os.path.join(fp, fn)): + raise LLMWareException(message=f"Path not found - {os.path.join(fp,fn)}") + + # * function declaration - add_one_pdf * + + # char * account_name + # char * library_name + # char * input_fp + # char * input_filename + # char * input_images_fp + # char * write_to_filename + # int user_block_size + + # if any issue loading module, will be captured at .get_module_pdf_parser() + _mod_pdf = Utilities().get_module_pdf_parser() + + pdf_handler = _mod_pdf.add_one_pdf + + pdf_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_int) + pdf_handler.restypes = c_int + + # prepare input variables + + t0 = time.time() + + # config options pulled from the Library object + if not self.account_name: + acct_name = "llmware" + + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + # fp = passed as parameter -> this is the input file path folder containing the .PDF docs to be parsed + + if not fp.endswith(os.sep): + fp += os.sep + + fp_c = create_string_buffer(fp.encode('ascii', 'ignore')) + + fn_c = create_string_buffer(fn.encode('ascii', 'ignore')) + + image_fp = self.parser_tmp_folder + if not image_fp.endswith(os.sep): + image_fp += os.sep + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + # prep parameters passed in the method invocation above + write_to_filename = "pdf_internal_test0.txt" + write_to_filename_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + # pull target block size from library parameters + + user_block_size = c_int(self.block_size_target_characters) # standard 400-600 + + logger.info("Parser - parse_one_pdf - starting pdf_parser ...") + + # main call into the pdf parser + + pages_created = pdf_handler(account_name, library_name, fp_c, fn_c, image_fp_c, + write_to_filename_c, user_block_size) + + logger.info(f"Parser - parse_one_pdf - completed pdf_parser - time taken: {time.time() - t0}") + + output = self.convert_parsing_txt_file_to_json(file_path=self.parser_tmp_folder, fn=write_to_filename) + + if len(output) > 0: + self.parser_output += output + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + def parse_one_office_deprecated_031_no_opts(self, fp, fn, save_history=True): + + """ Deprecated starting with llmware v 0.3.2 - entry point to parse one office document at + the selected file path and file name - fewer config options available. Will be removed in future + releases. """ + + # Designed for 'ad hoc' and 'unbound' quick parse of a single office document with no storage + # --output provided as list of Dicts in memory with same structure as parsing output + + # check that path exists + if not os.path.exists(os.path.join(fp, fn)): + raise LLMWareException(message=f"Path not found - {os.path.join(fp,fn)}") + + workspace_fp = self.parser_tmp_folder + + if not os.path.exists(workspace_fp): + os.mkdir(workspace_fp) + os.chmod(workspace_fp, 0o777) + + # safety check - will need to improve + expand for supporting windows path + if not workspace_fp.endswith(os.sep): + workspace_fp += os.sep + logger.warning("Parser - parse_one_office - workspace_fp did not end with trailing '/' " + "as expected by parser") + + # setup parser workspace + for z in range(0, 1): + + if os.path.exists(os.path.join(workspace_fp, str(z))): + shutil.rmtree(os.path.join(workspace_fp, str(z)), ignore_errors=True) + + if not os.path.exists(os.path.join(workspace_fp, str(z))): + os.mkdir(os.path.join(workspace_fp, str(z))) + os.chmod(os.path.join(workspace_fp, str(z)), 0o777) + + # * function declaration - add_one_office * + + # char * input_account_name + # char * input_library_name + # char * input_fp + # char * input_fn + # char * workspace_fp + # char * image_fp + # char * write_to_filename + + # if any issue loading module, will be captured at .get_module_office_parser() + _mod = Utilities().get_module_office_parser() + + main_handler = _mod.add_one_office + main_handler.argtypes = (c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p, c_char_p) + + main_handler.restype = c_int + + # three inputs - account_name // library_name // fp to web_dir - files to be processed + # prep each string: account_name = create_string_buffer(py_account_str.encode('ascii','ignore')) + + if not self.account_name: + self.account_name = "llmware" + + account_name = create_string_buffer(self.account_name.encode('ascii', 'ignore')) + library_name = create_string_buffer(self.library_name.encode('ascii', 'ignore')) + + if not fp.endswith(os.sep): + fp += os.sep + + fp_c = create_string_buffer(fp.encode('ascii', 'ignore')) + fn_c = create_string_buffer(fn.encode('ascii', 'ignore')) + + workspace_fp_c = create_string_buffer(workspace_fp.encode('ascii', 'ignore')) + + # image_fp = self.library.image_path + + # will need to fix this - C code expects trailing "/" + # image_fp = self.parser_tmp_folder # + "/" + image_fp = self.parser_image_folder + + if not image_fp.endswith(os.sep): + image_fp += os.sep + logger.debug("Adding '/' to image_fp as expected by c parser") + + image_fp_c = create_string_buffer(image_fp.encode('ascii', 'ignore')) + + write_to_filename = "office_internal_test0.txt" + write_to_fn_c = create_string_buffer(write_to_filename.encode('ascii', 'ignore')) + + # main call into office parser + pages_created = main_handler(account_name, library_name, fp_c, fn_c, workspace_fp_c, + image_fp_c, write_to_fn_c) + + # self.library.image_path + output = self.convert_parsing_txt_file_to_json(file_path=self.parser_tmp_folder, fn=write_to_filename) + + if len(output) > 0: + self.parser_output += output + + if save_history: + ParserState().save_parser_output(self.parser_job_id, self.parser_output) + + return output + + +class ImageParser: + + """ ImageParser for handling OCR of scanned documents - may be called directly, or through Parser. + Current implementation requires separate install of tesseract and pytesseract. """ + + def __init__(self, parser=None, library=None, text_chunk_size=600, look_back_range=300): + + self.parser = parser + + # defaults + self.text_chunk_size = text_chunk_size + self.look_back_range = look_back_range + + if library: + self.text_chunk_size = library.block_size_target_characters + 200 + self.look_back_range = 300 + + if parser and not library: + if parser.library: + self.text_chunk_size = parser.library.block_size_target_characters + 200 + self.look_back_range = 300 + + def process_ocr (self, dir_fp, fn, preserve_spacing=False): + + """ Process a single OCR file in 'dir_fp' and with filename 'fn'. """ + + try: + import pytesseract + from pytesseract.pytesseract import TesseractNotFoundError + except ImportError: + raise DependencyNotInstalledException("pytesseract") + + try: + text_out = pytesseract.image_to_string(os.path.join(dir_fp,fn)) + except TesseractNotFoundError as e: + raise DependencyNotInstalledException("tesseract") + + if not preserve_spacing: + text_out = text_out.replace("\n", " ") + + # will chop up the long text into individual blocks + text_chunks = TextChunker(text_chunk=text_out, + max_char_size=self.text_chunk_size, + look_back_char_range=self.look_back_range).convert_text_to_chunks() + + return text_chunks + + def ocr_to_single_text_file(self,fp): + + """ Runs OCR and converts image files into a single text file. """ + + # simple utility method to extract text directly from set of images in folder + # --will consolidate into a single text list + + try: + import pytesseract + from pytesseract.pytesseract import TesseractNotFoundError + except ImportError: + raise DependencyNotInstalledException("pytesseract") + + text_list_out = [] + scanned_files = os.listdir(fp) + + for docs in scanned_files: + + try: + text_out = pytesseract.image_to_string(os.path.join(fp,docs)) + except TesseractNotFoundError as e: + raise DependencyNotInstalledException("tesseract") + + text_out = text_out.replace("\n", " ") + logger.info(f"ImageParser - ocr_to_single_file - ocr text_out: {text_out}") + text_list_out.append(text_out) + + return text_list_out + + def process_pdf_by_ocr(self, input_fp, file): + + """ Handles special case of running page-by-page OCR on a scanned PDF document. """ + + text_output_by_page = [] + + try: + import pytesseract + from pytesseract.pytesseract import TesseractNotFoundError + except ImportError: + raise DependencyNotInstalledException("pytesseract") + + try: + from pdf2image import convert_from_path + from pdf2image.exceptions import PDFInfoNotInstalledError + except ImportError: + raise DependencyNotInstalledException("pdf2image") + + # decompose pdf into set of images by page + try: + images = convert_from_path(os.path.join(input_fp,file)) + except PDFInfoNotInstalledError as e: + raise DependencyNotInstalledException("popper") + for j, image in enumerate(images): + + # run ocr over page image + try: + text = pytesseract.image_to_string(image) + except TesseractNotFoundError as e: + raise DependencyNotInstalledException("tesseract") + + # will chop up the long text into individual blocks + text_chunks = TextChunker(text_chunk=text, + max_char_size=self.text_chunk_size, + look_back_char_range=self.look_back_range).convert_text_to_chunks() + + text_output_by_page.append(text_chunks) + + return text_output_by_page + + def exif_extractor(self, fp): + + """ Special utility to extract exif metadata from photos. """ + + # exif metadata is present in most photos, but not all + # if not a photo, it will not have exif data (e.g., camera standard) + + # most useful exif data is GPS coords, time_stamp and creator device (not always present) + + success_code = -1 + exif_table = {} + creator_device = {} + time_stamps = {} + + # PIL/Pillow required for EXIF image processing - must be installed separately + + try: + from PIL import Image + from PIL.ExifTags import TAGS, GPSTAGS + + except: + raise DependencyNotInstalledException("PIL") + + try: + img = Image.open(fp) + x = img._getexif() + except: + return success_code, creator_device, time_stamps, exif_table + + if x: + success_code = 1 + + for tag, value in x.items(): + + decoded = TAGS.get(tag, tag) + exif_table[decoded] = value + + if decoded == "Make": + creator_device.update({decoded: value}) + + if decoded == "Model": + creator_device.update({decoded: value}) + + if decoded.startswith("DateTime"): + time_stamps.update({decoded: value}) + + gps_info = {} + if exif_table: + if 'GPSInfo' in exif_table: + for key in exif_table['GPSInfo'].keys(): + decode = GPSTAGS.get(key, key) + gps_info[decode] = exif_table['GPSInfo'][key] + success_code = 1 + + return success_code, gps_info, creator_device, time_stamps, exif_table + + def convert_pdf_to_images_by_page(self, input_fp, output_fp, summary_text_fn= "text_summary.txt"): + + """ Converts scanned PDF file into Page-by-Page images. """ + + # converts pdf files into set of .png images by page + # --will process all of the pdf files in the input_fp + + input_files = os.listdir(input_fp) + + all_text = "" + + try: + import pytesseract + from pytesseract.pytesseract import TesseractNotFoundError + except ImportError: + raise DependencyNotInstalledException("pytesseract") + + try: + from pdf2image import convert_from_path + from pdf2image.exceptions import PDFInfoNotInstalledError + except ImportError: + raise DependencyNotInstalledException("pdf2image") + + for i, files in enumerate(input_files): + + ext = files.split(".")[-1] + if ext == "pdf": + try: + # decomposes pdf into set of image .png files + try: + images = convert_from_path(os.path.join(input_fp,files)) + except PDFInfoNotInstalledError as e: + raise DependencyNotInstalledException("poppler") + + for j, image in enumerate(images): + + # saves .png images in target output folder + fn = str(i) + "_" + str(j) + ".png" + image.save(os.path.join(output_fp,fn)) + try: + text = pytesseract.image_to_string(image) + except TesseractNotFoundError as e: + raise DependencyNotInstalledException("tesseract") + + all_text += text + # all_text += re.sub("[\n\r]"," ", text) + logger.info(f"update: ocr text out - {text}") + + logger.info(f"update: ocr converted- {i} - {files}") + all_text += "\n\n" + + except: + logger.error("error - could not convert pdf") + + f = open(input_fp + summary_text_fn, "w", encoding='utf-8') + f.write(all_text) + f.close() + + return summary_text_fn + + +class VoiceParser: + + """ VoiceParser handles wav files to convert into text blocks. """ + + def __init__(self, parser=None, library=None, text_chunk_size=600, look_back_range=300, + chunk_size=400, max_chunk_size=600, chunk_by_segment=True, remove_segment_markers=True, + real_time_progress=True): + + self.parser = parser + + # chunking parameters + self.chunk_size=chunk_size + self.max_chunk_size = max_chunk_size + self.chunk_by_segment = chunk_by_segment + self.remove_segment_markers = remove_segment_markers + self.real_time_progress = real_time_progress + + # defaults - for pure 'Text Chunking' - deprecating approach, but keeping as option for now + self.text_chunk_size = text_chunk_size + self.look_back_range = look_back_range + + if library: + self.text_chunk_size = library.block_size_target_characters + 200 + self.look_back_range = 300 + + if parser and not library: + if parser.library: + self.text_chunk_size = parser.library.block_size_target_characters + 200 + self.look_back_range = 300 + + self.speech_model = None + + from llmware.gguf_configs import GGUFConfigs + + # default model -> "whisper-cpp-base-english" + self.selected_speech_model_name = GGUFConfigs().get_config("whisper_default_model") + + # will update global GGUFConfigs based on real_time_progress preference + GGUFConfigs().set_config("whisper_cpp_realtime_display", self.real_time_progress) + + def voice_to_text(self,fp_input, fn, sr_input=16000): + + """Voice to text parsing conversion - looks up and calls the Model and gets inference response. """ + + from llmware.models import ModelCatalog + + self.speech_model = ModelCatalog().load_model(self.selected_speech_model_name) + + inference_dict = {"remove_segment_markers": self.remove_segment_markers} + + response = self.speech_model.inference(os.path.join(fp_input,fn),inference_dict=inference_dict) + + # response dictionary has several keys - "llm_response" | "segments" | "usage" + + # still exploring the best way to release memory once processing completed + self.speech_model.__dealloc__() + del self.speech_model + self.speech_model = None + + return response + + def add_voice_file(self, input_fp, fn): + + """ Parse voice file. """ + + output = [] + + # 16000 is standard default encoding rate for .wav -> may need further test/experiment + response = self.voice_to_text(input_fp, fn, 16000) + + if not self.chunk_by_segment: + + # this is initial strategy- deprecating for chunk_by_segment + text_out = response["text"] + # will chop up the long text into individual blocks + text_chunks = TextChunker(text_chunk=text_out, + max_char_size=self.text_chunk_size, + look_back_char_range=self.look_back_range).convert_text_to_chunks() + + for i, chunk in enumerate(text_chunks): + + new_entry = {"text": chunk, "start_time": 0, "end_time": 0, "speaker": "", "index": i, + "start_segment": 0, "end_segment": 0, "type": "text_only"} + + output.append(new_entry) + else: + # aggregate by segment within size parameters + if "segments" not in response: + logger.warning("VoiceParser - no 'segments' found in response from WhisperCPP.") + return [] + + char_counter = 0 + time_start = 0.0 + start_segment = 0 + t = "" + + for i, segment in enumerate(response["segments"]): + + current_segment_len = len(segment["text"]) + + if (char_counter + current_segment_len) >= self.chunk_size: + + # add to output list + + t += " " + segment["text"] + new_entry = {"text": t, "start_time": time_start, "end_time": segment["end"], + "speaker": "", "index": i, "start_segment": start_segment, "end_segment": i, + "type": "segments"} + output.append(new_entry) + t = "" + char_counter = 0 + time_start = segment["end"] + start_segment = i+1 + else: + if len(t) > 0 and ord(t[-1]) != 32: + t += " " + segment["text"] + else: + t += segment["text"] + + char_counter = len(t) + + # pick up last block of text + if len(t) > 0: + last_segment = response["segments"][-1] + new_entry = {"text": t, "start_time": time_start, "end_time": last_segment["end"], + "speaker": "", "index": len(response["segments"]), "start_segment": start_segment, + "end_segment": len(response["segments"]), "type": "segments"} + output.append(new_entry) + + return output + + +class TextParser: + + """ TextParser to parse .txt, .json, .csv, and .md files - can be called directly or through main Parser class. """ + + def __init__(self, parser=None, library=None, text_chunk_size=600, look_back_range=300): + + self.parser = parser + + # defaults + self.text_chunk_size = text_chunk_size + self.look_back_range = look_back_range + + if library: + self.text_chunk_size = library.block_size_target_characters + 200 + self.look_back_range = 300 + + if parser and not library: + if parser.library: + self.text_chunk_size = parser.library.block_size_target_characters + 200 + self.look_back_range = 300 + + def jsonl_file_handler (self, dir_fp,sample_file, key_list=None, interpret_as_table=False,separator="\n"): + + """ Parse JSON or JSONL file. """ + + # will extract each line in jsonl as separate sample + # --based on key_list and interpret_as_table + + output = [] + my_file = [] + + ft = sample_file.split(".")[-1].lower() + + if ft not in ["json", "jsonl"]: + logger.warning(f"TextParser - jsonl_file_parser did not find a recognized json/jsonl file type - " + f"{sample_file}") + return output + + if ft == "json": + my_file = json.load(open(os.path.join(dir_fp, sample_file), "r")) + + if ft == "jsonl": + file = open(os.path.join(dir_fp,sample_file), 'r', encoding='utf-8-sig',errors='ignore') + + output = [] + for i, lines in enumerate(file): + row_tmp = json.loads(lines) + my_file.append(row_tmp) + + file.close() + + if not key_list: + # as default, if no key_list, then look for "text" attribute in jsonl by default + key_list = ["text"] + + for i, lines in enumerate(my_file): + + row_tmp = lines + + if not interpret_as_table: + row_text = "" + for keys in key_list: + if keys in row_tmp: + row_text += str(row_tmp[keys]) + separator + output.append(row_text) + + else: + row_table = [] + for keys in key_list: + if keys in row_tmp: + row_table.append(str(row_tmp[keys])) + output.append(row_table) + + return output + + def text_file_handler (self, dir_fp, sample_file): + + """ Parse .txt file. """ + + text_out = open(os.path.join(dir_fp,sample_file), "r", encoding='utf-8-sig', errors='ignore').read() + + # will chop up the long text into individual text chunks + text_chunks = TextChunker(text_chunk=text_out, + max_char_size=self.text_chunk_size, + look_back_char_range=self.look_back_range).convert_text_to_chunks() + + return text_chunks + + def csv_file_handler (self, dir_fp,sample_file, interpret_as_table=True, delimiter=",", + encoding='utf-8-sig',errors='ignore', batch_size=1, separator="\t"): + + """ Parse .csv or .tsv file - depending upon separator, e.g., ',' or '\t' """ + + ft = sample_file.split(".")[-1].lower() + if ft == "tsv": + delimiter = "\t" + + # will split the table by rows and columns (\n for rows and ',' for cells in row) + t = Utilities.file_load(os.path.join(dir_fp,sample_file), + delimiter=delimiter, encoding=encoding, errors=errors) + + tables_out= [] + + if len(t) < batch_size: + tables_out = [t] + else: + table_chunks = len(t) // batch_size + + if batch_size > table_chunks * len(t): + # there is a remainder, so create one additional partial chunk with last set of rows + table_chunks += 1 + + starter = 0 + increment = 0 + + for x in range(0,table_chunks): + starter = starter + increment + increment = min(len(t)-starter, batch_size) + stopper = starter + increment + + if interpret_as_table: + tmp= t[starter:stopper] + else: + tmp = "" + for x in range(starter, stopper): + for y in range(0,len(t[x])): + tmp += str(t[x][y]) + separator + tmp = tmp[:-len(separator)] + tmp += "\n" + + tables_out.append(tmp) + + return tables_out + + +class WikiParser: + + """ WikiParser handles the retrieval and packaging of content from Wikipedia. """ + + def __init__(self, parser=None, library=None, text_chunk_size=600, look_back_range=300): + + self.wiki = WikiKnowledgeBase() + + self.parser = parser + self.library = library + + self.text_chunk_size = text_chunk_size + self.look_back_range = look_back_range + + if library: + self.text_chunk_size = self.library.block_size_target_characters + 200 + self.look_back_range = 300 + + if parser and not library: + if parser.library: + self.text_chunk_size = parser.library.block_size_target_characters + 200 + self.look_back_range = 300 + + def add_wiki_topic(self, topic, target_results=10): + + """ Parse a selected Wikipedia content by topic and requested target results. """ + + # used in both Parser / Library, as well as directly in Prompts (integrate as "Source" into Prompt) + + articles_output = [] + text_only = "" + blocks = [] + topic_query_results = self.wiki.search_wikipedia(topic,result_count=target_results, suggestion=False) + + text_chunks_all = [] + + for j, title in enumerate(topic_query_results): + article = self.wiki.get_article(title["title"]) + article.update({"topic": topic}) + articles_output.append(article) + + text_chunks = TextChunker(text_chunk=article["text"], + max_char_size=self.text_chunk_size, + look_back_char_range=self.look_back_range).convert_text_to_chunks() + + for i, chunk in enumerate(text_chunks): + new_block = {"file_source": title["title"], "page_num": max(1, i // 5), "text": chunk} + blocks.append(new_block) + + text_chunks_all += text_chunks + + topic_results = {"search_results": topic_query_results, "articles": articles_output, + "text_chunks": text_chunks_all, "blocks": blocks} + + return topic_results + + +class DialogParser: + + """ DialogParser handles parsing of dialog voice transcription, specifically for AWS currently. """ + + def __init__(self, parser=None, library=None, text_chunk_size=600, look_back_range=300): + + self.parser = parser + self.library = library + + self.text_chunk_size = text_chunk_size + self.look_back_range = look_back_range + + if library: + self.text_chunk_size = self.library.block_size_target_characters + 200 + self.look_back_range = 300 + + if parser and not library: + if parser.library: + self.text_chunk_size = parser.library.block_size_target_characters + 200 + self.look_back_range = 300 + + # currently only has support for AWS dialog format + self.supported_format_types = ["aws"] + + def parse_aws_json_file_format(self, input_folder, fn_json): + + """ Parse AWS JSON file. """ + + f = json.load(open(os.path.join(input_folder, fn_json), "r", encoding='utf-8-sig',errors='ignore')) + + # aws standard call transcript format: ["results"]["items"] -> key conversation elements to aggregate + # note: we will need many more documents for testing + # --possible that AWS call transcript has different formats and/or has evolved over time! + + block_output = [] + + # quick format check - will need to enhance over time + + format_validated = False + + if "results" in f: + if "items" in f["results"]: + format_validated = True + + # improve validation of format + user message back with link to AWS documents + if not format_validated: + logger.error("DialogParser currently only supports AWS Transcribe dialog format - For more " + "information, please see Amazon Web Services Transcription - " + "https://docs.aws.amazon.com/transcribe/latest/dg/how-input.html#how-it-works-output") + + return block_output + + # end - quick format check + + # speaker label conversation snippets + conversation_snippets = f["results"]["items"] + + if len(conversation_snippets) == 0: + # no results to parse + logger.error("DialogParser - unexpected parsing error - AWS JSON dialog transcript empty") + return block_output + + text= "" + current_speaker = "spk_0" + start_time = float(0) + end_time = float(0) + + for i, items in enumerate(conversation_snippets): + + if i == 0: + current_speaker = items["speaker_label"] + start_time = float(items["start_time"]) + end_time = float(items["end_time"]) + # initialize text with the first word + text="" + if "alternatives" in items: + if "content" in items["alternatives"][0]: + text = items["alternatives"][0]["content"] + + else: + # general case after first snippet + new_block = False + + # if found switch in speakers - write block and re-set + if "speaker_label" in items: + if items["speaker_label"] != current_speaker: + + new_block = True + + new_entry = {"speaker_name": current_speaker, + "speaker_id": current_speaker, "text": text, + "start_time": start_time, "stop_time": end_time} + + block_output.append(new_entry) + current_speaker = items["speaker_label"] + start_time = float(items["start_time"]) + end_time = float(items["end_time"]) + # re-initialize text with the first word of the new speaker + text = "" + if "alternatives" in items: + if "content" in items["alternatives"][0]: + text = items["alternatives"][0]["content"] + + if not new_block: + if "alternatives" in items: + if "content" in items["alternatives"][0]: + if items["type"] == "punctuation": + text += items["alternatives"][0]["content"] + else: + # general case - type = "pronunciation" [insert space] + text += " " + items["alternatives"][0]["content"] + + if "end_time" in items: + end_time = float(items["end_time"]) + + # pick up the last block, if any + if text: + new_entry = {"speaker_name": current_speaker, "speaker_id": current_speaker, "text": text, + "start_time": start_time, "stop_time": end_time} + block_output.append(new_entry) + + return block_output + diff --git a/llmware/prompts.py b/llmware/prompts.py new file mode 100644 index 0000000..627b67d --- /dev/null +++ b/llmware/prompts.py @@ -0,0 +1,2039 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The prompts module implements the Prompt class, which manages the inference process. This includes +pre-processing, executing, and post-processing of inferences and tracking the state of related inferences, +e.g. in conversational language models. + +The module also implements QualityCheck, and HumanInTheLoop classes, and leverages the Sources class (provided in the +util.py module). The QualityCheck class compares (i.e. verifies) the LLMs' response against the context information. +Finally, the HumanInTheLoop class provides mechanisms for reviews, which includes access to the prompt history +for corrections, as well as user ratings. +""" + +import statistics +import re +import time +import logging +import os + +from llmware.util import Utilities, CorpTokenizer, Sources +from llmware.web_services import YFinance +from llmware.resources import PromptState +from llmware.models import ModelCatalog, PromptCatalog, PyTorchLoader +from llmware.parsers import Parser +from llmware.retrieval import Query +from llmware.library import Library +from llmware.configs import LLMWareConfig, LLMWareException + +logger = logging.getLogger(__name__) +logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__)) + + +class Prompt: + + """Implements the actions of the prompt process, which includes the actions pre-processing, execution, + post-processing, and managing the state of related inferences. + + ``Prompt`` is responsible for pre-processing, executing, and post-processing of inferences and for + managing end-to-end state of a series of related inferences. + + Parameters + ---------- + llm_name : str, default=None + The name of the llm to be used. + + tokenizer : object, default=None + The tokenizer to use. The default is to use the tokenizer specified by the ``Utilities`` class. + + model_card : dict, default=None + A dictionary describing the model to be used. If the dictionary contains the key ``model_name``, + then this model will be used instead of the one set by ``llm_name``. In other words, ``model_card`` + overrides ``llm_name``. + + library : object, default=None + A ``Library`` object. + + account_name : str, default="llmware" + The name of the account to be used. This is one of the attributes of the prompt. + + prompt_id : int, default=None + The ID of the prompt. If a prompt ID is given, then the state of this prompt is loaded. Otherwise, a + new prompt ID is generated. This is part of the state of a prompt. + + save_state : bool, default=True + Actually, this is a dead variable and should be removed in a future release. + + llm_api_key : str, default=None + The API key that is used to load the large language model. + + llm_model : str, default=None + The name of the model to load. + + from_hf : bool, default=False + Indicates whether the model should be loaded from hugging face. + + prompt_catalog : object, default=None + An object of type ``PromptCatalog``. + + temperature : float, default=0.5 + Sets the temperature of the large language model. + + prompt_wrapper : str, default="human_bot" + Sets the prompt wrapper. Possible values are "alpaca", "human_bot", "chatgpt", "", "open_chat", + "hf_chat", and "chat_ml". + + instruction_following : bool, default=False + Sets whether the large language model should follow instructions. Note that this has an effect + if and only if the model specified has a version that is trained to follow instructions. + + """ + + def __init__(self, llm_name=None, tokenizer=None, model_card=None, library=None, account_name="llmware", + prompt_id=None, save_state=True, llm_api_key=None, llm_model=None, from_hf=False, + prompt_catalog=None, temperature=0.3, prompt_wrapper="human_bot", instruction_following=False): + + self.account_name = account_name + self.library = library + + # model specific attributes + self.model_card = model_card + self.tokenizer = tokenizer + + self.llm_model = None + self.llm_model_api_key = llm_api_key + self.llm_name = llm_name + self.llm_model_card = None + + if from_hf and llm_model: + + # will apply passed prompt wrapper and instruction_following settings + self.llm_model = ModelCatalog().load_hf_generative_model(llm_model, tokenizer, + prompt_wrapper=prompt_wrapper, + instruction_following=instruction_following) + + logger.debug(f"update: loading HF Generative model - {self.llm_model}") + + # default batch size, assuming all LLMs have min 2048 full context (50% in / 50% out) + self.context_window_size = 1000 + + if model_card: + + if "model_name" in model_card: + self.llm_model = ModelCatalog().load_model(model_card["model_name"], api_key=llm_api_key) + self.context_window_size = self.llm_model.max_input_len + self.llm_model_card = model_card + + # if passed llm model name, it will 'over-ride' the model_card if both passed + if llm_name: + + self.llm_model = ModelCatalog().load_model(llm_name, api_key=llm_api_key) + self.context_window_size = self.llm_model.max_input_len + + # inference parameters + self.temperature = temperature + self.prompt_type = "" + self.llm_max_output_len = 200 + + # state attributes + if prompt_id: + PromptState(self).load_state(prompt_id) + self.prompt_id = prompt_id + else: + new_prompt_id = PromptState(self).issue_new_prompt_id() + self.prompt_id = PromptState(self).initiate_new_state_session(new_prompt_id) + + logger.debug(f"update: Prompt - creating new prompt id - {new_prompt_id}") + + self.save_prompt_state = save_state + + # interaction_history is the main running 'active' tracker of current prompt history + # interaction_history is added by each 'register' invocation + # interaction_history can also be pulled from PromptState, or from database lookup + + self.interaction_history = [] + + # dialog tracker is an extract from the interaction history, consisting of running series of tuples: + # --"prompt" & "llm_response" response + + self.dialog_tracker = [] + + self.llm_state_vars = ["llm_response", "prompt", + "instruction", "usage", "time_stamp", "calling_app_ID", "account_name", + "prompt_id", "batch_id", "event_type", + # source/evidence + "evidence", "evidence_metadata", "biblio" + # fact-checking + "source_review", "comparison_stats", "fact_check", + # human-in-the-loop feedback + "human_feedback","human_assessed_accuracy", "human_rating", "change_log"] + + # prompt catalog options + if prompt_catalog: + self.pc = prompt_catalog + else: + self.pc = PromptCatalog() + + self.prompt_catalog = self.pc.get_all_prompts() + + # source materials - available for all prompts, passed as 'context' + # this is a 'stateful' list that aggregates and tracks all of the source materials added to the prompt + # each list entry consists of a dict with keys - "batch_id" | "text" | "batch_metadata" | "batch_stats" + # --batch_metadata is a list of metadata for each 'sub-source' integrated into the batch + # --batch_stats is a sub-list that tracks that # of elements in the batch_metadata + + self.source_materials = [] + + self.batch_separator = "\n" + + self.query_results = None + + self.model_catalog = ModelCatalog() + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + self.prompt_path = LLMWareConfig.get_prompt_path() + + # edge case - if llmware main path exists, but prompt path not created or deleted + if not os.path.exists(self.prompt_path): + os.mkdir(self.prompt_path) + os.chmod(self.prompt_path, 0o777) + + def load_model(self, gen_model,api_key=None, from_hf=False, trust_remote_code=False, + # new options added + use_gpu=True, sample=False, get_logits=False, + max_output=200, temperature=0.0, api_endpoint=None, **kwargs): + + """Load model into prompt object by selecting model name """ + + if api_key: + self.llm_model_api_key = api_key + + if not from_hf: + self.llm_model = self.model_catalog.load_model(gen_model, api_key=self.llm_model_api_key, + use_gpu=use_gpu, sample=sample, get_logits=get_logits, + max_output=max_output, temperature=temperature, + api_endpoint=api_endpoint, **kwargs) + if hasattr(self.llm_model, "model_card"): + self.llm_model_card = self.llm_model.model_card + + else: + + pt_loader = PyTorchLoader(api_key=api_key,trust_remote_code=trust_remote_code, custom_loader=None) + custom_hf_model = pt_loader.get_generative_model(gen_model) + hf_tokenizer = pt_loader.get_tokenizer(gen_model) + + # now, we have 'imported' our own custom 'instruct' model into llmware + self.llm_model = self.model_catalog.load_hf_generative_model(custom_hf_model, hf_tokenizer, + instruction_following=False, + prompt_wrapper="human_bot") + + # prepare 'safe name' without file paths + self.llm_model.model_name = re.sub("[/]","---",gen_model) + self.tokenizer = hf_tokenizer + + self.llm_name = gen_model + self.context_window_size = self.llm_model.max_input_len + self.llm_max_output_len = max_output + + return self + + def set_inference_parameters(self, temperature=0.5, llm_max_output_len=200): + + """ Convenience method to set inference parameters directly in prompt. """ + + self.temperature = temperature + self.llm_max_output_len = llm_max_output_len + return self + + def get_current_history(self, key_list=None): + + """ Will return selected state vars from current prompt session, based on key list """ + + if not key_list: + key_list = self.llm_state_vars + + output_dict = {} + for i, keys in enumerate(key_list): + output_dict.update({keys: []}) + for j, entries in enumerate(self.interaction_history): + if keys in entries: + output_dict[keys].append(entries[keys]) + + return output_dict + + def clear_history(self): + + """ Removes elements from interaction history """ + + self.interaction_history = [] + self.dialog_tracker = [] + return self + + def clear_source_materials(self): + + """ Clears the source materials from the prompt to start with fresh set of sources """ + self.source_materials = [] + return self + + def register_llm_inference (self, ai_dict, prompt_id=None, trx_dict=None): + + """ Registers the llm inference to prompt state """ + + if not prompt_id: + prompt_id = self.prompt_id + + # update elements from interaction + ai_dict.update({"prompt_id": prompt_id}) + ai_dict.update({"event_type": "inference"}) + ai_dict.update({"human_feedback": ""}) + ai_dict.update({"human_assessed_accuracy": ""}) + + # if trx_dict passed -> append key/value pairs into ai_dict + if isinstance(trx_dict, dict): + for key,value in trx_dict.items(): + ai_dict.update({key:value}) + + # captures new interaction into the interaction history + logger.debug(f"update: ai_dict getting registered - {ai_dict['event_type']}") + + PromptState(self).register_interaction(ai_dict) + new_dialog = {"user": ai_dict["prompt"], "bot": ai_dict["llm_response"]} + self.dialog_tracker.append(new_dialog) + + return ai_dict + + def lookup_llm_trx_all (self): + + """ Look up saved llm transactions persisted to file in prompt history """ + + ai_trx_list = PromptState(self).full_history() + return ai_trx_list + + def load_state(self, prompt_id, clear_current_state=True): + + """ Loads an existing prompt history state by prompt_id from prompt history """ + + PromptState(self).load_state(prompt_id,clear_current_state=clear_current_state) + for entries in self.interaction_history: + self.dialog_tracker.append({"user": entries["prompt"], "bot": entries["llm_response"]}) + + return self + + def save_state(self): + + """ Saves the state of the prompt and writes to prompt history file """ + + PromptState(self).save_state(self.prompt_id) + return self + + def lookup_by_prompt_id (self, prompt_id): + + """ Look up specific prompts by prompt_id """ + + ai_trx_list = PromptState(self).lookup_by_prompt_id(prompt_id) + return ai_trx_list + + def lookup_ai_trx_with_filter(self, filter_dict): + + """ Look up prompts by filter dictionary """ + + ai_trx_list = PromptState(self).lookup_prompt_with_filter(filter_dict) + return ai_trx_list + + def add_source_new_query(self, library, query=None, query_type="semantic", result_count=10): + + """ Attach a new source to a prompt object by running a new query against a library. """ + + # step 1 - run selected query against library + query_results = Query(library).query(query,query_type=query_type, result_count=result_count, results_only=True) + + # step 2 - package query_results directly as source, loaded to prompt, and packaged as 'llm context' + sources = Sources(self).package_source(query_results,aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_new_query.") + + return sources + + def add_source_query_results(self, query_results): + + """ Attach a new source to a prompt object by passing directly the query results from a previous query. """ + + # example use - run a query directly, and then 'add' the query results to a prompt + # query_results = Query(self.library).semantic_query("what is the duration of the non-compete clause?") + # prompter = Prompt().load_model("claude-instant-v1",api_key="my_api_key") + # sources = prompter.add_source_query_results(query_results["results"]) + + sources = Sources(self).package_source(query_results,aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_query_results.") + + return sources + + def add_source_library(self, library_name, account_name="llmware"): + + """ Attach a new source to a prompt object by passing an entire library - note: only recommended if the library + consists of a very small number of documents. """ + + # example use - created a small library with a few key documents in a previous step + # my_lib.add_documents(fp) + # sources = prompter.add_source_library("my_lib") + + lib = Library().load_library(library_name, account_name=account_name) + query_results = Query(lib).get_whole_library() + + sources = Sources(self).package_source(query_results, aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_library.") + + return sources + + def add_source_wikipedia(self, topic, article_count=3, query=None): + + """ Attach a wikipedia source to a prompt object by selecting a topic and count of requested articles. """ + + # step 1 - get wikipedia article + output = Parser().parse_wiki([topic],write_to_db=False,target_results=article_count) + + if query: + if output: + output = Utilities().fast_search_dicts(query, output, remove_stop_words=True) + + for i, entries in enumerate(output): + logger.debug(f"update: source entries - {i} - {entries}") + + # step 2 - package wiki article results as source, loaded to prompt, and packaged as 'llm context' + sources = Sources(self).package_source(output,aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_wikipedia.") + + return sources + + def add_source_yahoo_finance(self, ticker=None, key_list=None): + + """ Attach a source to a prompt object by selecting a ticker from Yahoo Finance. """ + + # example: primary use is to quickly grab a factset about a specific company / stock ticker + # and 'inject' real-time, up-to-date fact set into the prompt to minimize hallucination risk + + fin_info = YFinance().ticker(ticker).info + + logger.debug(f"update: fin_info - {fin_info}") + + output = "" + if key_list: + for keys in key_list: + if keys in fin_info: + output += keys + " : " + str(fin_info[keys]) + self.batch_separator + else: + for keys, values in fin_info.items(): + output += keys + " : " + str(values) + self.batch_separator + + results = {"file_source": "yfinance-" + str(ticker), "page_num": "na", "text": output} + + logger.debug(f"update: yfinance results - {results}") + + # step 2 - package as source + sources = Sources(self).package_source([results], aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_yahoo_finance.") + + return sources + + def add_source_website(self, url, query=None): + + """ Attach a website source to a prompt object by identifying the url name. """ + + # get website content + output = Parser().parse_website(url,write_to_db=False,max_links=3) + + if query: + if output: + output = Utilities().fast_search_dicts(query, output, remove_stop_words=True) + + if not output: output = [] + + sources = Sources(self).package_source(output, aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_website.") + + return sources + + def add_source_document(self, input_fp,input_fn, query=None): + + """ Attach a document directly to a prompt object by passing the folder path and file name of the source + document, and an optional query filter. """ + + # example: intended for use to rapidly parse and add a document (of any type) from local file to a prompt + + output = Parser().parse_one(input_fp,input_fn) + + # run in memory filtering to sub-select from document only items matching query + if query: + if output: + output = Utilities().fast_search_dicts(query, output, remove_stop_words=True) + + if not output: output = [] + + sources = Sources(self).package_source(output, aggregate_source=True) + + if not sources["text_batch"]: + logger.warning("No source added in .add_source_document.") + + return sources + + def add_source_last_interaction_step(self): + + """ Adds the last interaction step directly into the source to enable 'interactive dialog'. """ + + interaction= "" + if len(self.dialog_tracker) > 0: + interaction += self.dialog_tracker[-1]["user"] + "\n" + self.dialog_tracker[-1]["bot"] + "\n" + + interaction_source = [{"text": interaction, "page_num":0, "file_source":"dialog_tracker"}] + + sources = Sources(self).package_source(interaction_source, aggregate_source=True) + + # enables use of 'prompt_with_sources' + if not sources["text_batch"]: + logger.warning("No source added in .add_source_last_interaction_step.") + + return sources + + def review_sources_summary(self): + + """ Review the sources and provide summary. """ + + # Source metadata for each entry - ["batch_id", "text", "metadata", "biblio", "batch_stats", + # "batch_stats.tokens", "batch_stats.chars", "batch_stats.samples"] + + source_summary_output = [] + for i, sources in enumerate(self.source_materials): + + # add biblio to output + new_entry = {"batch_id": sources["batch_id"], "batch_stats": sources["batch_stats"], + "biblio": sources["biblio"]} + + source_summary_output.append(new_entry) + + return source_summary_output + + def verify_source_materials_attached(self): + + """ Verifies if source materials attached. Returns True if text present in source materials, else False. """ + + source_materials_attached = False + + if len(self.source_materials) > 0: + + for sources in self.source_materials: + if "text" in sources: + if len(sources["text"]) > 0: + source_materials_attached = True + break + + return source_materials_attached + + def prompt_with_source(self, prompt, prompt_name=None, source_id_list=None, first_source_only=True, + max_output=None, temperature=None, verbose=False): + + """ Inference method - uses the prepared source, along with prompt/question, and calls loaded model. """ + + # this method is intended to be used in conjunction with sources as follows: + # prompter = Prompt().load_model("claude-instant-v1", api_key=None) + # source = prompter.add_source (....) + # response = prompter.prompt_with_source("what is the stock price of XYZ?") + # + # if multiple loaded sources, then the method will automatically call the model several times + # --user can select either 'call once' with first_source_only = True + # --OR ... by selecting specific sources by their batch_id, + # e.g., source_id_list = [0,1,5] would iterate through sources 0, 1, 5 + + response_list = [] + response_dict = {} + + if prompt_name: + self.prompt_type = prompt_name + + if max_output: + self.llm_max_output_len = max_output + + if temperature: + self.temperature = temperature + + # this method assumes a 'closed context' with set of preloaded sources into the prompt + # if len(self.source_materials) == 0: + if not self.verify_source_materials_attached(): + + logger.warning("No source materials attached to the Prompt. " + "Running prompt_with_source inference without source may lead to unexpected results.") + + response_dict = self.prompt_main(prompt,prompt_name=self.prompt_type,context="", + register_trx=False,temperature=temperature) + + # by default - prompt_with_source returns a list of response dictionaries + return [response_dict] + + # this is the 'default' and will use the first batch of source material only + if first_source_only: + + response_dict = self.prompt_main(prompt,prompt_name=self.prompt_type, + context=self.source_materials[0]["text"], + register_trx=False, temperature=temperature) + + # add details on the source materials to the response dict + if "metadata" in self.source_materials[0]: + response_dict.update({"evidence_metadata": self.source_materials[0]["metadata"]}) + + if "biblio" in self.source_materials[0]: + response_dict.update({"biblio": self.source_materials[0]["biblio"]}) + + response_list.append(response_dict) + + else: + # if first_source_only is false, then run prompts with all of the sources available + for i, batch in enumerate(self.source_materials): + if source_id_list: + + if i in source_id_list: + response_dict = self.prompt_main(prompt,prompt_name=self.prompt_type, + context=self.source_materials[i]["text"], + register_trx=False, temperature=temperature) + + # add details on the source materials to the response dict + if "metadata" in self.source_materials[i]: + response_dict.update({"evidence_metadata": self.source_materials[i]["metadata"]}) + + if "biblio" in self.source_materials[i]: + response_dict.update({"biblio": self.source_materials[i]["biblio"]}) + + response_list.append(response_dict) + + else: + + response_dict = self.prompt_main(prompt, prompt_name=self.prompt_type, + context=self.source_materials[i]["text"], + register_trx=False, temperature=temperature) + + # add details on the source materials to the response dict + if "metadata" in self.source_materials[i]: + response_dict.update({"evidence_metadata": self.source_materials[i]["metadata"]}) + + if "biblio" in self.source_materials[i]: + response_dict.update({"biblio": self.source_materials[i]["biblio"]}) + + response_list.append(response_dict) + + # log progress of iterations at info level + if verbose: + logger.info(f"update: prompt_with_sources - iterating through source batches - {i} - {response_dict['llm_response']}") + + # register inferences in state history, linked to prompt_id + for l, llm_inference in enumerate(response_list): + + logger.debug (f"update: llm inference - {l} - {len(response_list)} - {llm_inference}") + + self.register_llm_inference(llm_inference) + + return response_list + + def select_prompt_from_catalog(self, prompt_name): + + """ Selects a prompt style from the catalog. """ + + if prompt_name in self.pc.list_all_prompts(): + self.prompt_type = prompt_name + else: + raise LLMWareException(message=f"Prompt - select_prompt_from_catalog - " + f"unable to find selected prompt in " + f"catalog - {prompt_name}") + return self + + def prompt_from_catalog(self, prompt, context=None, prompt_name=None, inference_dict=None): + + """ Inference method - runs a prompt by loading a specific prompt style from the catalog. """ + + if prompt_name not in self.pc.list_all_prompts(): + raise LLMWareException(message=f"Prompt - prompt_from_catalog - could " + f"not find selected prompt in catalog - " + f"{prompt_name}") + + # self.llm_model.add_prompt_engineering= prompt_name + response = self.prompt_main(prompt,context=context, prompt_name=prompt_name,inference_dict=inference_dict) + + return response + + def number_or_none(self, prompt, context=None): + + """ Inference method - convenience method using 'number_or_none' prompt style instruction. """ + + output = self.prompt_from_catalog(prompt, context=context,prompt_name="number_or_none") + return output + + def summarize_with_bullets(self, prompt, context, number_of_bullets=5): + + """ Inference method - convenience method using 'summarize_with_bullets' prompt style and configurable + number of 'bullets' requested. """ + + # useful 'out of the box' summarize capability with ability to parameterize the number_of_bullets + # note: most models are 'approximately' accurate when specifying a number of bullets + + inference_dict = {"number_of_bullets": number_of_bullets} + output = self.prompt_from_catalog(prompt, context=context,prompt_name="summarize_with_bullets", + inference_dict=inference_dict) + + return output + + def yes_or_no(self, prompt, context): + + """ Inference method - convenience method using 'yes_no' prompt style. """ + + # useful classification prompt, assumes prompt is a question that expects a "yes" or "no" answer + response = self.prompt_from_catalog(prompt, context=context,prompt_name="yes_no") + + return response + + def completion(self, prompt, temperature=0.7, target_len=200): + + """ Inference method - convenience method for a basic text completion. """ + + self.llm_model.temperature = temperature + self.llm_model.ai_max_output_len = target_len + + response = self.prompt_from_catalog(prompt, prompt_name="completion") + + return response + + def multiple_choice(self, prompt, context, choice_list): + + """ Inference method - prepares a multiple choice question prompt, using prompt, context and choice list. """ + + prompt += "\nWhich of the following choices best answers the question - " + for i, choice in enumerate(choice_list): + prompt += "(" + chr(65+i) + ") " + choice + ", " + + if prompt.endswith(", "): + prompt = prompt[:-2] + "?" + + response = self.prompt_from_catalog(prompt, context=context, prompt_name="multiple_choice") + + return response + + def xsummary(self, context, number_of_words=20): + + """ Inference method - uses 'xsummary' prompt style and configurable number of requested words for + short summaries.""" + + # provides an 'extreme summary', e.g., 'xsum' with ability to parameterize the number of words + # --most models are reasonably accurate when asking for specific number of words + + prompt="" + inference_dict = {"number_of_words": number_of_words} + response = self.prompt_from_catalog(prompt, context=context, prompt_name="xsummary",inference_dict=inference_dict) + + return response + + def title_generator_from_source (self, prompt, context=None, title_only=True): + + """ Inference method - uses 'report_title' prompt style to produce titles based on prompt and context. """ + + response = self.prompt_from_catalog(prompt, context=context,prompt_name="report_title") + + if title_only: + return response["llm_response"] + + return response + + def prompt_main (self, prompt, prompt_name=None, context=None, call_back_attempts=1, calling_app_id="", + prompt_id=0,batch_id=0, trx_dict=None, selected_model= None, register_trx=False, + inference_dict=None, max_output=None, temperature=None): + + """ Main inference method to execute inference on loaded model. """ + + usage = {} + + if not prompt_name: + + # pull from .add_prompt_engineering state + if self.llm_model.add_prompt_engineering: + prompt_name = self.llm_model.add_prompt_engineering + + else: + # defaults + if context: + prompt_name = "default_with_context" + else: + prompt_name = "default_no_context" + + if selected_model: + self.llm_model = self.model_catalog.load_model(selected_model) + + if temperature: + self.temperature = temperature + + self.llm_model.temperature = self.temperature + + if max_output: + self.llm_max_output_len = max_output + + self.llm_model.target_requested_output_tokens = self.llm_max_output_len + self.llm_model.add_context = context + self.llm_model.add_prompt_engineering = prompt_name + + # if the loaded model is function_calling, then execute a function call instead of inference + use_fc = False + if hasattr(self.llm_model, "fc_supported"): + use_fc = self.llm_model.fc_supported + + if use_fc: + output_dict = self.llm_model.function_call(context, params=[prompt]) + output = output_dict["llm_response"] + + else: + output_dict = self.llm_model.inference(prompt, inference_dict=inference_dict) + + output = output_dict["llm_response"] + + if isinstance(output,list): + output = output[0] + + # triage process - if output is ERROR code, then keep trying up to parameter- call_back_attempts + # by default - will not attempt to triage, e.g., call_back_attempts = 1 + # --depending upon the calling function, it can decide the criticality and # of attempts + + if output == "/***ERROR***/": + # try again + attempts = 1 + + while attempts < call_back_attempts: + + # wait 5 seconds to try back + time.sleep(5) + + # exact same call to inference + output_dict = self.llm_model.inference(prompt) + + output = output_dict["llm_response"] + # if list output, then take the string from the first output + if isinstance(output, list): + output = output[0] + + # keep trying until not ERROR message found + if output != "/***ERROR***/": + break + + attempts += 1 + + # if could not triage, then present "pretty" error output message + if output == "/***ERROR***/": + if "error_message" in output_dict: + output = output_dict["error_message"] + else: + output = "AI Output Not Available" + + # strip & which are used by some models as end of text marker + if not use_fc: + output = str(output).replace("","") + output = str(output).replace("","") + + if "usage" in output_dict: + usage = output_dict["usage"] + + output_dict = {"llm_response": output, "prompt": prompt, + "evidence": context, + "instruction": prompt_name, "model": self.llm_model.model_name, + "usage": usage, + "time_stamp": Utilities().get_current_time_now("%a %b %d %H:%M:%S %Y"), + "calling_app_ID": calling_app_id, + "rating": "", + "account_name": self.account_name, + "prompt_id": prompt_id, + "batch_id": batch_id, + } + + if context: + evidence_stop_char = len(context) + else: + evidence_stop_char = 0 + output_dict.update({"evidence_metadata": [{"evidence_start_char":0, + "evidence_stop_char": evidence_stop_char, + "page_num": "NA", + "source_name": "NA", + "doc_id": "NA", + "block_id": "NA"}]}) + + if register_trx: + self.register_llm_inference(output_dict,prompt_id,trx_dict) + + return output_dict + + def _doc_summarizer_old_works(self, query_results, max_batch_size=100, max_batch_cap=None,key_issue=None): + + """ Deprecated - summarizes a batch of query results - will be removed in the future, but kept for backwards + compatibility, and if useful for a particular summarization task. """ + + # runs core summarization loop thru document + + big_batches = len(query_results) // max_batch_size + # if there was a 'remainder', then run one additional loop ... + # ... this also picks up the 'normal' case of query_results < max_batch_size + if len(query_results) > big_batches * max_batch_size: + big_batches += 1 + + response = [] + + if max_batch_cap: + if big_batches > max_batch_cap: + + logger.warning(f"warning: Prompt document summarization - you have requested a " + f"maximum cap of {max_batch_cap} batches - so truncating the batches " + f"from {big_batches} to " + f"the cap requested - note that content will be missing as a result.") + + big_batches = max_batch_cap + + for x in range(0,big_batches): + + qr = query_results[x*max_batch_size:min((x+1)*max_batch_size,len(query_results))] + + source = self.add_source_query_results(qr) + + if key_issue: + response += self.prompt_with_source(key_issue, prompt_name="summarize_with_bullets_w_query", + first_source_only=False) + else: + placeholder_issue = "What are the main points?" + response += self.prompt_with_source(placeholder_issue,prompt_name="summarize_with_bullets", + first_source_only=False) + + return response + + def _doc_summarizer(self, query_results, max_batch_cap=None,key_issue=None): + + """ Runs Core summarization loop through a selected document. """ + + response = [] + + source = self.add_source_query_results(query_results) + + if max_batch_cap: + if len(self.source_materials) > max_batch_cap: + + logger.warning(f"warning: Prompt document summarization - you have requested a " + f"maximum cap of {max_batch_cap} batches - so truncating the batches from " + f"{len(self.source_materials)} to" + f"the cap requested - note that content will be missing as a result.") + + self.source_materials = self.source_materials[0:max_batch_cap] + + if key_issue: + response += self.prompt_with_source(key_issue, prompt_name="summarize_with_bullets_w_query", + first_source_only=False) + else: + placeholder_issue = "What is a list of the main points?" + response += self.prompt_with_source(placeholder_issue,prompt_name="default_with_context", + first_source_only=False) + + return response + + def summarize_document(self, input_fp,input_fn, query=None, text_only=True, max_batch_cap=10, + key_issue=None): + + """ Input is a path to a document file (fp, fn), which will then be parsed in line, searched if there is a + query provided, then summarize and return a document summary as output. """ + + output = Parser().parse_one(input_fp,input_fn) + + # run in memory filtering to sub-select from document only items matching query + if query: + output = Utilities().fast_search_dicts(query, output, remove_stop_words=True) + + response = self._doc_summarizer(output, key_issue=key_issue, max_batch_cap=max_batch_cap) + + if text_only: + # return only text + output_text = "" + + for i, entries in enumerate(response): + if "llm_response" in entries: + output_text += entries["llm_response"] + "\n" + + return output_text + + else: + return response + + def summarize_document_fc(self, fp, fn, topic="key points", query=None, text_only=True, max_batch_cap=15, + summary_model="slim-summary-tool", real_time_update=True): + + """ New document summarization method built on slim-summary-tool. """ + + if real_time_update: + logger.info(f"update: Prompt - summarize_document_fc - document - {fn}") + + # note: when loading model, context window is automatically set based on model + self.load_model(summary_model, temperature=0.0, sample=False) + + self.llm_max_output_len = 150 + + if not query: + sources = self.add_source_document(fp, fn) + else: + sources = self.add_source_document(fp, fn, query=query) + + if len(self.source_materials) > max_batch_cap: + self.source_materials = self.source_materials[0:max_batch_cap] + + if real_time_update: + + logger.info(f"update: Prompt - summarize_document_fc - number of source batches - " + f"{len(self.source_materials)}") + + key_points = [] + + responses = self.prompt_with_source(topic, first_source_only=False, verbose=True) + + for i, resp in enumerate(responses): + + for point in resp["llm_response"]: + if point not in key_points: + if point.strip(): + if not point.strip().startswith("Not Found"): + key_points.append(point) + + return key_points + + def summarize_document_from_library(self, library, doc_id=None, filename=None, query=None, + text_only=True,max_batch_cap=10): + + """ Returns a document summary - based on a selected document ID from a library. """ + + # need to handle error + if not doc_id and not filename: + placeholder = "no file received" + return -1 + + if doc_id: + key = "doc_ID" + value = doc_id + else: + key = "file_source" + value = filename + + if not query: + if not isinstance(value,list): + value = [value] + + query_results = Query(library).filter_by_key_value_range(key, value) + + else: + if isinstance(value,list): + if len(value) > 0: + value = value[0] + filter_dict = {key:value} + query_results = Query(library).text_query_with_custom_filter(query,filter_dict,result_count=20) + + response = self._doc_summarizer(query_results, max_batch_cap=max_batch_cap) + + if text_only: + # return only text + output_text = "" + + for i, entries in enumerate(response): + if "llm_response" in entries: + output_text += entries["llm_response"] + "\n" + + return output_text + + else: + return response + + def summarize_multiple_responses(self, list_of_response_dict=None, response_id_list=None): + + """ Summarizes multiple responses from previous inferences as a 'second-level' summary. """ + + batch = None + + if list_of_response_dict: + batch = list_of_response_dict + elif response_id_list: + batch = [] + for response_id in response_id_list: + batch += PromptState(self).lookup_by_prompt_id + + if not batch: + batch = self.interaction_history + + # batch of response dictionaries -> need to aggregate the llm_responses- and run prompt + aggregated_response_dict = {} + + return aggregated_response_dict + + def select_among_multiple_responses(self, list_of_response_dict=None, response_id_list=None): + + """ Aggregates multiple previous responses and passes as a 'second-level' inference to select the best + answer. """ + + batch = None + + if list_of_response_dict: + batch = list_of_response_dict + elif response_id_list: + batch = [] + for response_id in response_id_list: + batch += PromptState(self).lookup_by_prompt_id + + if not batch: + batch = self.interaction_history + + # batch of response dictionaries -> need to aggregate the llm_responses- and run prompt + aggregated_response_dict = {} + + return aggregated_response_dict + + def evidence_check_numbers(self, response): + + """ Post Inference Processing - runs analysis of the numbers in the llm_response and attempts to verify + the values of those numbers in the source materials. + + Returns an updated list of response dictionaries, enriched with "fact_check" key. """ + + # expect that response is a list of response dictionaries + if isinstance(response, dict): + response = [response] + + response_out = [] + + for i, response_dict in enumerate(response): + qc = QualityCheck(self).fact_checker_numbers(response_dict) + + response_dict.update({"fact_check": qc}) + response_out.append(response_dict) + + return response_out + + def evidence_check_sources(self, response): + + """ Post Inference Processing - runs analysis of the llm_response and uses statistical token-matching + with the source materials to try to identify a smaller 'snippet' that is the most likely source with + metadata of file and page number. + + Returns an updated list of response dictionaries, enriched with 'source_review' key. """ + + # expect that response is a list of response dictionaries + if isinstance(response, dict): + response = [response] + + response_out = [] + for i, response_dict in enumerate(response): + qc = QualityCheck(self).source_reviewer(response_dict) + + response_dict.update({"source_review": qc}) + response_out.append(response_dict) + + return response_out + + def evidence_comparison_stats(self, response): + + """ Post Inference Processing - runs analysis of the llm_response and uses statistical token-matching + with the source materials to provide an overall comparison 'match' level which can be a good + quantitative indicator if the model output has hallucinated or deviated materially from the source. + + Returns an updated list of response dictionaries, enriched with 'comparison_stats' key. """ + + # expect that response is a list of response dictionaries + if isinstance(response, dict): + response = [response] + + response_out = [] + for i, response_dict in enumerate(response): + qc = QualityCheck(self).token_comparison(response_dict) + + response_dict.update({"comparison_stats": qc}) + response_out.append(response_dict) + + return response_out + + def classify_not_found_response(self, response_list,parse_response=True,evidence_match=True,ask_the_model=False): + + """ Post Inference Processing - takes a list of response dictionaries as input, and then runs tests to + validate if the llm_response appears to be 'not found'.""" + + output_response_all = [] + + if isinstance(response_list,dict): + response_list = [response_list] + + for i, response_dict in enumerate(response_list): + output_response_all.append(self._classify_not_found_one_response(response_dict, + parse_response=parse_response, + evidence_match=evidence_match, + ask_the_model=ask_the_model)) + + return output_response_all + + def _classify_not_found_one_response(self, response_dict, parse_response=True, evidence_match=True, ask_the_model=False): + + """ Internal utility helper to classify a single response.""" + + output_response = {} + nf = [] + + if parse_response: + nf1 = QualityCheck(self).classify_not_found_parse_llm_response(response_dict) + output_response.update({"parse_llm_response": nf1}) + if nf1 not in nf: + nf.append(nf1) + + if evidence_match: + nf2 = QualityCheck(self).classify_not_found_evidence_match(response_dict) + output_response.update({"evidence_match": nf2}) + if nf2 not in nf: + nf.append(nf2) + + if ask_the_model: + nf3 = QualityCheck(self).classify_not_found_ask_the_model(response_dict) + output_response.update({"ask_the_model": nf3}) + if nf3 not in nf: + nf.append(nf3) + + if len(nf) == 0: + logger.warning("error: Prompt().classify_not_response() expects at least one of the tests to be marked" + "as True - none of the tests were executed - please try again with one test as 'True'") + + return output_response + + # simple case - all of the tests are conforming + if len(nf) == 1: + output_response.update({"not_found_classification": nf[0]}) + else: + output_response.update({"not_found_classification": "undetermined"}) + + return output_response + + def send_to_human_for_review(self, output_path=None, output_fn=None): + + """ Exports the current prompt interaction to a CSV for follow-up review by a person. """ + + output = HumanInTheLoop(prompt=self).export_current_interaction_to_csv(output_path=output_path,report_name=output_fn) + return output + + def apply_user_ratings(self, ratings_dict): + + """ Adds a human rating to a response dictionary - useful to upstream applications to enable and capture + user input. """ + + output = HumanInTheLoop(prompt=self).add_or_update_human_rating(self.prompt_id,ratings_dict) + return output + + def apply_user_corrections(self, updates_dict): + + """ Enables a user to manually update llm_responses as second-level human-in-the-loop review in upstream + application. """ + + output = HumanInTheLoop(prompt=self).update_llm_response_record(self.prompt_id,updates_dict,keep_change_log=True) + return output + + +class QualityCheck: + """Implements the validation between the output of the LLM and the context used to generate the response, + which is used by the ``Prompt`` class. + + ``QualityCheck`` allows for the comparison of LLM generated responses with the context that was used to + create the response. Concretely, it is quality verifying mechanism used by the ``Prompt`` class. + One use case is to verify that reported numbers in the response appear in the context. + + Parameters + ---------- + prompt : object, default=None + An object of type ``Prompt``. + + Examples + ---------- + >>> import os + >>> from llmware.setup import Setup + >>> from llmware.library import Library + >>> from llmware.prompts import Prompt + >>> library = Library().create_new_library('prompt_with_sources') + >>> sample_files_path = Setup().load_sample_files(over_write=False) + >>> parsing_output = library.add_files(os.path.join(sample_files_path, "Agreements")) + >>> prompter = Prompt().load_model('llmware/bling-1b-0.1') + >>> prompter.add_source_document(os.path.join(sample_files_path, "Agreements"), 'Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf') + >>> result = prompter.prompt_with_source(prompt='What is the base salery amount?', prompt_name='default_with_context') + >>> result[0]['llm_response'] + ' $1,000,000.00' + >>> ev_numbers = prompter.evidence_check_numbers(result) + >>> ev_numbers[0].keys() + dict_keys(['llm_response', 'prompt', 'evidence', 'instruction', 'model', + 'usage', 'time_stamp', 'calling_app_ID', 'rating', 'account_name', + 'prompt_id', 'batch_id', 'evidence_metadata', 'biblio', 'event_type', + 'human_feedback', 'human_assessed_accuracy', + 'fact_check']) + >>> ev_numbers[0]['fact_check'] + [{'fact': 'detail.', 'status': 'Not Confirmed', 'text': '', 'page_num': '', 'source': ''}] + >>> ev_sources = prompter.evidence_check_sources(result) + >>> ev_sources[0].keys() + dict_keys(['llm_response', 'prompt', 'evidence', 'instruction', 'model', + 'usage', 'time_stamp', 'calling_app_ID', 'rating', 'account_name', + 'prompt_id', 'batch_id', 'evidence_metadata', 'biblio', 'event_type', + 'human_feedback', 'human_assessed_accuracy', + 'fact_check', 'source_review']) + >>> ev_sources[0]['source_review'] + [] + >>> ev_stats = prompter.evidence_comparison_stats(result) + >>> ev_stats[0].keys() + dict_keys(['llm_response', 'prompt', 'evidence', 'instruction', 'model', + 'usage', 'time_stamp', 'calling_app_ID', 'rating', 'account_name', + 'prompt_id', 'batch_id', 'evidence_metadata', 'biblio', 'event_type', + 'human_feedback', 'human_assessed_accuracy', 'fact_check', 'source_review', 'comparison_stats']) + >>> ev_stats[0]['comparison_stats'] + {'percent_display': '0.0%', 'confirmed_words': [], + 'unconfirmed_words': ['1000000.00'], 'verified_token_match_ratio': 0.0, + 'key_point_list': [{'key_point': ' $1,000,000.00', 'entry': 0, 'verified_match': 0.0}]} + """ + def __init__(self, prompt=None): + + self.llm_response = None + self.evidence = None + self.evidence_metadata= None + self.add_markup = False + + self.prompt = prompt + + # add instruction + self.instruction = None + + self.comparison_stats = {} + self.fact_check = {} + self.ner_fact_check = {} + self.source_review = {} + + def review (self, response_dict, add_markup=False, review_numbers=True, comparison_stats=True, + source_review=True, instruction=None): + + """ Input as list of response dictionaries, and output is response dictionaries enriched with review keys. """ + + self.llm_response = response_dict["llm_response"] + self.evidence= response_dict["evidence"] + self.evidence_metadata = response_dict["evidence_metadata"] + self.add_markup = add_markup + + # add instruction + self.instruction = instruction + + # review - main entry point into Quality Check - runs several methods for output + + if comparison_stats: + self.comparison_stats = self.token_comparison (response_dict) + + if review_numbers: + self.fact_check = self.fact_checker_numbers(response_dict) + + if source_review: + self.source_review = self.source_reviewer(response_dict) + + return self + + def fact_checker_numbers (self, response_dict): + + """ Utility function to compare and match number values in llm_response with input source materials. In most + cases, this function should be accessed through the prompt evidence methods rather than calling directly. """ + + ai_gen_output = response_dict["llm_response"] + evidence = response_dict["evidence"] + evidence_metadata = response_dict["evidence_metadata"] + + # looks for numbers only right now + llm_response_markup = "" + fact_check = [] + + ai_numbers = [] + ai_numbers_token_tracker = [] + ai_numbers_char_tracker = [] + + confirmations = [] + unconfirmations = [] + + tokens = ai_gen_output.split(" ") + percent_on = -1 + char_counter = 0 + + for i, tok in enumerate(tokens): + + tok_len = len(tok) + + # minimal cleaning of tokens + + # remove bullet point + if len(tok) > 0: + if ord(tok[-1]) == 8226: + tok = tok[:-1] + + if len(tok) > 1: + if tok.startswith("\n"): + tok = tok[1:] + + if tok.endswith("\n"): + tok = tok[:-1] + + if tok.endswith(",") or tok.endswith(".") or tok.endswith("-") or tok.endswith(";") or \ + tok.endswith(")") or tok.endswith("]"): + tok = tok[:-1] + + if tok.startswith("$") or tok.startswith("(") or tok.startswith("["): + tok = tok[1:] + + if tok.endswith("%"): + tok = tok[:-1] + percent_on = 1 + + tok = re.sub("[,-]","",tok) + # look for integer numbers - will not find floats + if Utilities().isfloat(tok): + + if percent_on == 1: + tok_fl = float(tok) / 100 + # turn off + percent_on = -1 + else: + tok_fl = float(tok) + ai_numbers.append(tok_fl) + ai_numbers_token_tracker.append(i) + ai_numbers_char_tracker.append((char_counter,char_counter+tok_len)) + + char_counter += tok_len + 1 + + # iterate thru all of the numbers generated - and look for match in evidence + found_confirming_match = [] + tokens = evidence.split(" ") + evidence_char_counter = 0 + percent_on = -1 + current_str_token = "" + + for x in range(0, len(ai_numbers)): + match_tmp = -1 + match_token = -1 + + percent_on = -1 + for i, tok in enumerate(tokens): + + tok_len = len(tok) + + if tok.endswith("\n"): + tok = tok[:-1] + + # current_str_token = tok + + if tok.endswith(",") or tok.endswith(".") or tok.endswith("-") or tok.endswith(";") or \ + tok.endswith(")") or tok.endswith("]"): + tok = tok[:-1] + + if tok.startswith("$") or tok.startswith("(") or tok.startswith("["): + tok = tok[1:] + + if tok.endswith("%"): + tok = tok[:-1] + percent_on = 1 + + tok = re.sub("[,-]","",tok) + + # current_str_token set to the 'cleaned' tok + current_str_token = tok + + if Utilities().isfloat(tok): + tok = float(tok) + if percent_on == 1: + tok = tok / 100 + # turn off + percent_on = -1 + + if tok == ai_numbers[x]: + + match_token = i + + if i > 10: + start = i-10 + else: + start = 0 + + if i+10 < len(tokens): + stop = i+10 + else: + stop = len(tokens) + + context_window = " ... " + for j in range(start,stop): + context_window += tokens[j] + " " + context_window = re.sub("[\n\r]","",context_window) + context_window += " ... " + + # insert page_num - future update + # default - set to the last batch + minibatch = len(evidence_metadata)-1 + + for m in range(0,len(evidence_metadata)): + + starter = evidence_metadata[m]["evidence_start_char"] + stopper = evidence_metadata[m]["evidence_stop_char"] + if starter <= char_counter <= stopper: + minibatch = m + break + + # set default as "NA" - will update once confirmed found in evidence_metadata below + page_num = "NA" + source_fn = "NA" + + if len(evidence_metadata[minibatch]) > 1: + if "page_num" in evidence_metadata[minibatch]: + page_num = evidence_metadata[minibatch]["page_num"] + + if "source_name" in evidence_metadata[minibatch]: + source_fn = evidence_metadata[minibatch]["source_name"] + + new_fact_check_entry = {"fact": current_str_token, + "status": "Confirmed", + "text": context_window, + "page_num": page_num, + "source": source_fn} + fact_check.append(new_fact_check_entry) + + confirmations.append(current_str_token) + + match_tmp = 1 + break + + evidence_char_counter += tok_len + 1 + + if match_tmp == -1: + + # change here - replace 'current_str_token' + new_fact_check_entry = {"fact": str(ai_numbers[x]), + "status": "Not Confirmed", + "text": "", + "page_num": "", + "source": ""} + + fact_check.append(new_fact_check_entry) + unconfirmations.append(current_str_token) + + # provide markup highlighting confirmations and non-confirmations + confirm_updates = [] + + # add_markup feature turned to OFF by default + # -- may be reworked or deleted in future releases + add_markup = False + + if add_markup: + for i,f in enumerate(fact_check): + + char_slice = ai_numbers_char_tracker[i] + + # if not confirmed status, then markup as "unconfirm" + markup_entry = [i, ai_numbers_char_tracker[i], "unconfirm"] + + # test to update mark_up entry to "confirm" + if len(f) > 1: + if "status" in f: + if f["status"] == "Confirmed": + markup_entry = [i, ai_numbers_char_tracker[i], "confirm"] + + confirm_updates.append(markup_entry) + + confirm_updates = sorted(confirm_updates, key=lambda x:x[0], reverse=True) + + ai_output_markup = ai_gen_output + + for c in confirm_updates: + + output_tmp = ai_output_markup + + if c[2] == "confirm": + ai_output_markup = output_tmp[0:c[1][0]] + " " + ai_output_markup += output_tmp[c[1][0]:c[1][1]] + " " + ai_output_markup += output_tmp[c[1][1]:] + else: + ai_output_markup = output_tmp[0:c[1][0]] + " " + ai_output_markup += output_tmp[c[1][0]:c[1][1]] + " " + ai_output_markup += output_tmp[c[1][1]:] + + # fact_check.update({"confirmations": confirmations}) + # fact_check.update({"unconfirmations": unconfirmations}) + # fact_check.update({"ai_web_markup": ai_output_markup}) + + # note: ai_web_markup not passed + + return fact_check + + def source_reviewer (self, response_dict): + + """ Utility function to compare and match llm_response with input source materials. In most + cases, this function should be accessed through the prompt evidence methods rather than calling directly. """ + + ai_tmp_output = response_dict["llm_response"] + evidence_batch = response_dict["evidence"] + evidence_metadata = response_dict["evidence_metadata"] + add_markup = False + + min_th = 0.25 + conclusive_th = 0.75 + min_match_count = 3 + + # remove numbers from source review match ??? + c = CorpTokenizer(remove_stop_words=True, one_letter_removal=True, remove_punctuation=True, + remove_numbers=False, lower_case=False) + + c2 = CorpTokenizer(remove_stop_words=False, one_letter_removal=False, remove_punctuation=True, + remove_numbers=False, lower_case=False) + + # alt: ai_tmp_output = re.sub("[()\"\u201d\u201c]"," ", ai_tmp_output) + ai_tokens = c.tokenize(ai_tmp_output) + ai_token_len = len(ai_tokens) + + if ai_token_len == 0: + # rare case - no ai output, so no need to do any further work + empty_results = [] + return empty_results + + matching_evidence_score = [] + for x in range(0, len(evidence_metadata)): + match = 0 + ev_match_tokens = [] + + ev_starter = evidence_metadata[x]["evidence_start_char"] + ev_stopper = evidence_metadata[x]["evidence_stop_char"] + + local_text = evidence_batch[ev_starter:ev_stopper] + # alt: local_text = re.sub("[()\"\u201d\u201c]", "", local_text) + evidence_tokens_tmp = c2.tokenize(local_text) + # alt: evidence_tokens_tmp = local_text.split(" ") + + for tok in ai_tokens: + for i, etoks in enumerate(evidence_tokens_tmp): + + # \n left by tokenization + etoks = etoks.strip() + + if etoks: + if tok.lower() == etoks.lower(): + match += 1 + ev_match_tokens.append(i) + break + + match_score = match / ai_token_len + + # min threshold to count as source -> % of total or absolute # of matching tokens + if match_score > min_th or len(ev_match_tokens) > min_match_count: + matching_evidence_score.append([match_score, x, ev_match_tokens, evidence_tokens_tmp, evidence_metadata[x]["page_num"], evidence_metadata[x]["source_name"], evidence_metadata[x]["doc_id"], evidence_metadata[x]["block_id"]]) + + mes = sorted(matching_evidence_score, key=lambda x: x[0], reverse=True) + + sources_output = [] + text_output = [] + + if len(mes) > 3: + top_sources = 3 + else: + top_sources = len(mes) + + for m in range(0, top_sources): + + page_num = mes[m][4] + source_name = mes[m][5] + doc_id = mes[m][6] + block_id = mes[m][7] + + # text_snippet = "Page {}- ... ".format(str(page_num)) + text_snippet = "" + + median_token = int(statistics.median(mes[m][2])) + if median_token >= 10: + starter = median_token - 10 + else: + starter = 0 + + if median_token + 10 < len(mes[m][3]): + stopper = median_token + 10 + else: + stopper = len(mes[m][3]) + + for y in range(starter, stopper): + text_snippet += str(mes[m][3][y]) + " " + + # text_snippet += " ... " + + text_snippet = re.sub("[\n\r]", " ... ", text_snippet) + + if text_snippet not in text_output: + text_output.append(text_snippet) + + # new_output = {"text": text_snippet, "match_score": mes[m][0],"source": evidence_metadata[mes[m][1]]} + new_output = {"text": text_snippet, "match_score": mes[m][0], "source": source_name, + "page_num": page_num, "doc_id": doc_id, "block_id": block_id} + + sources_output.append(new_output) + + if mes[m][0] > conclusive_th: + # found conclusive source -> no need to look for others + break + + return sources_output + + def token_comparison (self, response_dict): + + """ Utility function to perform token-level comparison in llm_response with input source materials. In most + cases, this function should be accessed through the prompt evidence methods rather than calling directly. """ + + # --applies different rules by instruction, e.g., yes-no exclude + # --if number in output, looks to handle 'word numbers' + float value comparison + # --if multiple points in output, will run comparison separately against each "key point" + + ai_output_text = response_dict["llm_response"] + evidence_batch = response_dict["evidence"] + evidence_metadata = response_dict["evidence_metadata"] + + yes_no = False + key_point_output_list = [] + + if self.instruction == "yes_no": + yes_no = True + + key_point_list = [ai_output_text] + + c = CorpTokenizer(remove_stop_words=True, remove_numbers=False,one_letter_removal=True, remove_punctuation=False) + evidence_tokens = c.tokenize(evidence_batch) + + # iterate thru each key point and analyze comparison match + confirmed_match_agg = [] + unmatched_agg = [] + ai_tokens_agg = [] + + evidence_with_numbers = "" + evidence_numbers_list = [] + + for i, kp in enumerate(key_point_list): + + ai_tokens = c.tokenize(kp) + ai_tokens_agg += ai_tokens + + # skip any empty kp + if len(ai_tokens) > 0: + + confirmed_match = [] + unmatched = [] + + for tok in ai_tokens: + match = -1 + + # sharpen matching rules for dollar amounts + if tok.endswith("."): + tok = tok[:-1] + + # only remove "." or "," if at the end + tok = re.sub("[,();$\"\n\r\t\u2022\u201c\u201d]","",tok) + + float_check_on = Utilities().isfloat(tok) + + run_compare = True + + if float_check_on: + if not evidence_with_numbers: + + evidence_with_numbers, evidence_numbers_list, \ + token_index_location = Utilities().replace_word_numbers(evidence_batch) + + for ev_num in evidence_numbers_list: + try: + if float(ev_num) == float(tok): + confirmed_match.append(tok) + match = 1 + run_compare = False + except: + pass + + if run_compare: + for etoks in evidence_tokens: + + # mirrors check in the evidence + if etoks.endswith("."): + etoks = etoks[:-1] + + etoks = re.sub("[(),;$\n\r\t\"\u2022\u201c\u201d]","",etoks) + + # removed lemmatizer and other approximate string matches - look for exact match + if tok == etoks: + confirmed_match.append(tok) + match = 1 + break + + # add token compare check if number -> look for numeric equality (even if strings different) + if float_check_on: + if Utilities().isfloat(etoks): + if float(tok) == float(etoks): + confirmed_match.append(tok) + match = 1 + break + + if match == -1: + # no duplicates + if tok not in unmatched: + unmatched.append(tok) + + # create new entry for kp + match = len(confirmed_match) / len(ai_tokens) + new_entry = {"key_point": kp, "entry": len(key_point_output_list), "verified_match": match} + key_point_output_list.append(new_entry) + unmatched_agg += unmatched + confirmed_match_agg += confirmed_match + + # match_percent = 0.0 + match_percent = "{0:.1f}%".format(0.0) + match_fr = 0.0 + + if len(ai_tokens_agg) > 0: + + match_fr = len(confirmed_match_agg) / len(ai_tokens_agg) + if match_fr > 1.0: + match_fr = 1.0 + match_percent = "{0:.1f}%".format((match_fr * 100)) + + # how to handle, if at all? + if yes_no and match_fr == 0: + no_action_for_now = 0 + + comparison_stats = {"percent_display": match_percent, + "confirmed_words": confirmed_match_agg, + "unconfirmed_words": unmatched_agg, + "verified_token_match_ratio": match_fr, + "key_point_list": key_point_output_list} + + return comparison_stats + + def classify_not_found_parse_llm_response(self, response_dict): + + """Simple, but reasonably accurate way to classify as "not found" - especially with "not found" instructions + --(1) most models will follow the "not found" instruction and this will be the start of the response + --(2) if a model gets confused and does not provide any substantive response, then this will get flagged too + """ + + # minimal cleaning of response output + llm_response = response_dict["llm_response"] + llm_response_cleaned = re.sub("[;!?•(),.\n\r\t\u2022]", "", llm_response).strip().lower() + + # first test: if no content in 'cleaned' response + if not llm_response_cleaned: + return True + + # second test: if response starts with 'not found' + if llm_response_cleaned.lower().startswith("not found"): + return True + + return False + + def classify_not_found_evidence_match (self, response_dict, verified_token_match_threshold=0.25): + + """ Objective of this method is to classify a LLM response as "not found" + --this is a key requirement of 'evidence-based' retrieval augmented generation + Note on output: "True" - indicates that classification of 'Not Found' + "False" - indicates not 'Not Found' - in other words, use as a valid response + """ + + if "comparison_stats" not in response_dict: + comparison_stats = self.token_comparison(response_dict) + else: + comparison_stats = response_dict["comparison_stats"] + + verified_token_match = comparison_stats["verified_token_match_ratio"] + + # simple threshold passed as parameter - assumes 0.25 as baseline + # --e.g., if there is less than 1 in 4 tokens verified in evidence, SKIP + # --we could make this higher filter, but occasionally might exclude a valid answer in different format + + llm_response = response_dict["llm_response"] + llm_response_cleaned = re.sub("[;!?•(),.\n\r\t\u2022]", "", llm_response).strip().lower() + + # carve-out "yes" | "no" answers - special case - will not having 'matching tokens' in evidence + if llm_response_cleaned in ["yes", "yes.", "no","no."]: + return False + + if verified_token_match < verified_token_match_threshold: + return True + + return False + + def classify_not_found_ask_the_model(self, response_dict, selected_model_name=None, model_api_key=None): + + """ Experimental method to 'ask the model' to classify its own response - some models very effective + at doing this - others perform poorly - please handle with care. """ + + if not selected_model_name: + selected_model_name = self.prompt.llm_name + model_api_key = self.prompt.llm_model_api_key + + new_prompt = Prompt().load_model(selected_model_name,api_key=model_api_key) + new_response = new_prompt.prompt_from_catalog(prompt="", context=response_dict["llm_response"], + prompt_name="not_found_classifier") + + llm_response = new_response["llm_response"] + llm_response_cleaned = re.sub("[;!?•(),.\n\r\t\u2022]", "", llm_response).strip().lower() + + if llm_response_cleaned.startswith("yes"): + return True + + if llm_response_cleaned.startswith("no"): + return False + + # if the test is inconclusive, then it returns False + + return False + + +class HumanInTheLoop: + """Implements the human reviewing features, which are used by the ``Prompt`` class. + + ``HumanInTheLoop`` provides utilities to extract prompt history states for secondary level review. + Currently, this includes sending an interaction to a human for review, modifying the response of + the model, and adding user ratings to an interaction. + + Parameters + ---------- + prompt : object + An object of type ``Prompt``. + + prompt_id_list : list, default=None + A list of prompt ids. + + Examples + ---------- + >>> import os + >>> from llmware.setup import Setup + >>> from llmware.library import Library + >>> from llmware.prompts import Prompt, HumanInTheLoop + >>> library = Library().create_new_library('prompt_with_sources') + >>> sample_files_path = Setup().load_sample_files(over_write=False) + >>> parsing_output = library.add_files(os.path.join(sample_files_path, "Agreements")) + >>> prompt = Prompt().load_model('llmware/bling-1b-0.1') + >>> prompt.add_source_document(os.path.join(sample_files_path, "Agreements"), 'Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf') + >>> result = prompt.prompt_with_source(prompt='What is the base salery amount?', prompt_name='default_with_context') + >>> csv_metadata = HumanInTheLoop(prompt).export_current_interaction_to_csv() + >>> csv_metadata + {'report_name': 'interaction_report_Sun Mar 10 17:16:01 2024.csv', + 'report_fp': '/home/user/llmware_data/prompt_history/interaction_report_Sun Mar 10 17:16:01 2024.csv', + 'results': 1} + """ + def __init__(self, prompt, prompt_id_list=None): + + self.prompt= prompt + self.user_rating_keys = ["human_rating", "human_feedback", "human_assessed_accuracy", "change_log"] + + def export_interaction_to_csv(self, prompt_id_list=None, output_path=None, report_name=None): + + """Input a list of one or more prompt_ids and dump to csv for user to review and edit """ + + output = PromptState(self.prompt).generate_interaction_report(prompt_id_list, + output_path=output_path, + report_name=report_name) + + return output + + def export_current_interaction_to_csv(self, output_path=None, report_name=None): + + """ this method will take the current interaction state and dump to csv for user to review and edit """ + + output = PromptState(self.prompt).generate_interaction_report_current_state(output_path=output_path, + report_name=report_name) + + return output + + def import_updated_csv(self, fp, fn, prompt_id): + + """ Not implemented yet. """ + + # allows corrections to be uploaded by csv spreadsheet and corrections made in the history + + return 0 + + def add_or_update_human_rating (self, prompt_id, rating_dict): + + """ Adds and updates human rating and feedback to a selected response dictionary. """ + + rating = -1 + accuracy = "" + feedback = "" + + f = {"prompt_id": prompt_id} + + if "human_rating" in rating_dict: + rating = int(rating_dict["human_rating"]) + + if "human_feedback" in rating_dict: + feedback = rating_dict["human_feedback"] + + if "human_assessed_accuracy" in rating_dict: + accuracy = rating_dict["human_assessed_accuracy"] + + update_dict = {"human_rating": rating, "human_feedback": feedback, "human_assessed_accuracy": accuracy} + + PromptState(self).update_records(prompt_id, f, update_dict) + + return 0 + + def update_llm_response_record(self,prompt_id, update_dict,keep_change_log=True): + + """ Provide more general update, including corrections, to a response dictionary 'post-human-review.' """ + + # as default option, preserve the current values in a change_log list + # --over time, we can evaluate whether to capture more metadata about the change, roll-back, etc. + + if keep_change_log: + # get original record - will save in "change_log" list below changing + current_record = list(PromptState(self).lookup_by_prompt_id(prompt_id=prompt_id)) + # current_record = list(coll.find(f)) + + if len(current_record) == 1: + current_dict = {} + for keys in update_dict: + if keys in current_record[0]: + # this is what will be saved in the list of 'change log' events within the record + current_dict.update({keys:current_record[0][keys], + "time_stamp":Utilities().get_current_time_now()}) + + if "change_log" in current_record[0]: + change_log = current_record[0]["change_log"] + else: + change_log = [] + change_log.append(current_dict) + update_dict.update({"change_log": change_log}) + + # save and update records + confirmation = PromptState(self).update_records(prompt_id,f,update_dict) + + return confirmation + + + diff --git a/llmware/requirements.txt b/llmware/requirements.txt new file mode 100644 index 0000000..88e3403 --- /dev/null +++ b/llmware/requirements.txt @@ -0,0 +1,13 @@ +# core requirements for main llmware operations and use cases +# please see also requirements_extras.txt + +# installed with standard pip +numpy>=1.23.2 +requests +huggingface-hub>=0.19.4 +tokenizers>=0.15.0 +boto3>=1.24.53 +colorama==0.4.6 + + + diff --git a/llmware/requirements_extras.txt b/llmware/requirements_extras.txt new file mode 100644 index 0000000..75e3b33 --- /dev/null +++ b/llmware/requirements_extras.txt @@ -0,0 +1,31 @@ +# requirements_extras - optional libraries most often used in conjunction with llmware +# these libraries will be installed with the welcome_to_llmware.sh script and are used in many examples + +# to use pytorch models (including embedding models) +torch>=1.13.1 +transformers>=4.36.0 +einops>=0.7.0 + +# to use milvus - note: we have seen compatibility issues with 2.5.4, so recommend 2.5.0 +pymilvus<=2.5.1 + +# to use mongo db +pymongo>=4.7.0 + +# to use postgres and pg vector +psycopg-binary==3.1.17 +psycopg==3.1.17 +pgvector==0.2.4 + +# for voice-to-text processing +soundfile>=0.12.0 +soxr>=0.5.0 + +# misc +Wikipedia-API>=0.6.0 +openai>=1.0 +datasets>=2.15.0 +yfinance>=0.2.38 +chromadb>=0.4.22 +streamlit + diff --git a/llmware/resources.py b/llmware/resources.py new file mode 100644 index 0000000..ac6d1d6 --- /dev/null +++ b/llmware/resources.py @@ -0,0 +1,5304 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The resources module implements the text index databases that are used as the foundation for creating a +Library in LLMWare, and a wide range of supporting methods, including text query retrieval, library card + management, tracking of embedding progress and status, and the ability to create custom tables. The text index + is used as the 'master' source of aggregating and access unstructured information that has been parsed and + organized into Library collections. + +Currently, llmware supports MongoDB, Postgres, and SQLite as text index databases, and supports the use of both +Postgres and SQLIte for creation of custom (SQL) tables. + +""" + +import platform, os, ast, json, csv, uuid, re, random, logging, sys, time +from datetime import datetime +from threading import Thread + +try: + from pymongo import MongoClient, ReturnDocument + from bson import ObjectId + import pymongo + from pymongo.errors import ConnectionFailure +except ImportError: + pass + +try: + import boto3 + from botocore import UNSIGNED + from botocore.config import Config + from botocore.exceptions import ClientError +except ImportError: + pass + +from llmware.configs import (LLMWareConfig, PostgresConfig, LLMWareTableSchema, + SQLiteConfig, AWSS3Config, LLMWareException) + +# new imports +try: + import sqlite3 +except ImportError: + pass + +try: + import psycopg +except ImportError: + pass + +logger = logging.getLogger(__name__) +logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__)) + + +class CollectionRetrieval: + + """CollectionRetrieval is primary class abstraction to handle all queries to underlying Text Index Database. + All calling functions should use CollectionRetrieval, which will, in turn, route to the correct DB resource """ + + def __init__(self, library_name, account_name="llmware",db_name=None,custom_table=False, custom_schema=False): + + self.library_name = library_name + self.account_name = account_name + + self.supported_collection_db = LLMWareConfig().get_supported_collection_db() + + # allow direct pass of db name + if db_name: + self.active_db = db_name + else: + self.active_db = LLMWareConfig().get_active_db() + + self._retriever = None + + if self.active_db in self.supported_collection_db: + + if self.active_db == "mongo": + self._retriever = MongoRetrieval(self.library_name, account_name=account_name, + custom_table=custom_table, custom_schema=custom_schema) + + if self.active_db == "postgres": + self._retriever = PGRetrieval(self.library_name, account_name=account_name, + custom_table=custom_table, custom_schema=custom_schema) + + if self.active_db == "sqlite": + self._retriever = SQLiteRetrieval(self.library_name, account_name=account_name, + custom_table=custom_table, custom_schema=custom_schema) + + else: + raise LLMWareException(message=f"CollectionRetrieval - collection database " + f"is not supported - {self.active_db}") + + def test_connection(self): + """Pings database and confirms valid connection""" + return self._retriever.test_connection() + + def safe_name(self, input_name): + """ Checks if collection name valid for db resource """ + return self._retriever.safe_name(input_name) + + def lookup(self, key,value): + """lookup returns a list of dictionary entries - generally a list of 1 entry for 'lookup'""" + return self._retriever.lookup(key,value) + + def embedding_key_lookup(self, key, value): + return self._retriever.embedding_key_lookup(key,value) + + def get_whole_collection(self): + """Retrieves whole collection, e.g., filter {} or SELECT * FROM {table}- will return a Cursor object""" + return self._retriever.get_whole_collection() + + def basic_query(self, query): + """Simple text query passed to the text index""" + return self._retriever.basic_query(query) + + def filter_by_key(self, key, value): + """Filter_by_key accepts a key string, corresponding to a column in the DB, and matches to a value""" + return self._retriever.filter_by_key(key, value) + + def text_search_with_key_low_high_range(self, query, key, low, high, key_value_dict=None): + """Text search with a key, such as page or document number, and matches entries in a range of 'low' to 'high'""" + return self._retriever.text_search_with_key_low_high_range(query, key, low, high, key_value_dict=key_value_dict) + + def text_search_with_key_value_range(self, query, key, value_range_list, key_value_dict=None): + """Text search with added filter of confirming that a key is in the selected value_range list + with option for any number of further constraints passed as optional key_value_dict""" + return self._retriever.text_search_with_key_value_range(query, key, value_range_list, + key_value_dict=key_value_dict) + + def text_search_with_key_value_dict_filter(self, query, key_value_dict): + """Text search with with {key:value} filter added""" + return self._retriever.text_search_with_key_value_dict_filter(query, key_value_dict) + + def get_distinct_list(self, key): + """Returns distinct list of elements in collection by key""" + return self._retriever.get_distinct_list(key) + + def filter_by_key_dict(self, key_dict): + """Filters by key dictionary""" + return self._retriever.filter_by_key_dict(key_dict) + + def filter_by_key_value_range(self, key, value_range): + """Filters by key value range""" + return self._retriever.filter_by_key_value_range(key, value_range) + + def filter_by_key_ne_value(self, key, value): + """Filters by key not equal to selected value""" + return self._retriever.filter_by_key_ne_value(key, value) + + def count_documents(self, filter_dict): + """Counts entries returned by filter dict""" + return self._retriever.count_documents(filter_dict) + + def close(self): + """Close underlying DB connection - handled by underlying DB resource""" + return self._retriever.close() + + # 2 specific reads for embedding + def embedding_job_cursor(self, new_embedding_key, doc_id=None): + """Handles end-to-end retrieval of text blocks selected for embedding & returns cursor""" + return self._retriever.embedding_job_cursor(new_embedding_key,doc_id=doc_id) + + def count_embedded_blocks(self, embedding_key): + """Counts the number of blocks to be created for an embedding job""" + return self._retriever.count_embedded_blocks(embedding_key) + + def direct_custom_query(self, query_filter): + """Applies the custom query directly to the DB and returns the results""" + return self._retriever.direct_custom_query(query_filter) + + def list_all_tables(self): + """Get list of all collections on the database""" + return self._retriever.list_all_tables() + + def get_schema(self, table_name): + """Return schema for selected table""" + return self._retriever.get_schema(table_name) + + +class CollectionWriter: + + """CollectionWriter is the main class abstraction for writing, editing, and deleting new elements to the + underlying text collection index - calling functions should use CollectionWriter, which will route and manage + the connection to the underlying DB resource""" + + def __init__(self, library_name, account_name="llmware", db_name=None, custom_table=False, custom_schema=None): + + self.library_name = library_name + self.account_name = account_name + + self.supported_collection_db = LLMWareConfig().get_supported_collection_db() + + if db_name: + self.active_db = db_name + else: + self.active_db = LLMWareConfig().get_active_db() + + self._writer = None + + if self.active_db in self.supported_collection_db: + + if self.active_db == "mongo": + self._writer = MongoWriter(self.library_name, account_name=self.account_name, custom_table=custom_table, + custom_schema=custom_schema) + + if self.active_db == "postgres": + self._writer = PGWriter(self.library_name, account_name=self.account_name, custom_table=custom_table, + custom_schema=custom_schema) + + if self.active_db == "sqlite": + self._writer = SQLiteWriter(self.library_name, account_name=self.account_name, custom_table=custom_table, + custom_schema=custom_schema) + + else: + raise LLMWareException(message=f"CollectionWriter - collection database " + f"is not supported - {self.active_db}") + + def build_text_index(self): + """Builds text index using db-specific methods""" + self._writer.build_text_index() + return 1 + + def check_if_table_build_required(self): + """Checks if table build required- returns True if table build required, e.g., no table found + and building table schema is required by the DB resource""" + + build_table = self._writer.check_if_table_build_required() + + return build_table + + def create_table(self, table_name, schema): + """Creates table""" + return self._writer.create_table(table_name, schema) + + def write_new_record(self, new_record): + """Inserts new record to the DB resource - unpacks and validates the new_record dict, if required """ + return self._writer.write_new_record(new_record) + + def write_new_parsing_record(self, new_record): + """Inserts new parsing record to the DB resource """ + return self._writer.write_new_parsing_record(new_record) + + def destroy_collection(self, confirm_destroy=False): + """Drops the collection associated with the library""" + return self._writer.destroy_collection(confirm_destroy=confirm_destroy) + + #TODO: may be able to remove - called only by Library.update_block + #suggest preserving: it is materially useful - and in use + def update_block(self, doc_id, block_id, key, new_value, default_keys): + """Updates specific row, based on doc_id and block_id""" + return self._writer.update_block(doc_id, block_id, key, new_value, default_keys) + + def update_one_record(self, filter_dict, key, new_value): + """Updates one record selected by filter_dict""" + return self._writer.update_one_record(filter_dict, key, new_value) + + #TODO: may be able to remove - not called + """ + def update_many_records(self, filter_dict, key, new_value): + # Updates multiple records selected by filter_dict + return self._writer.update_many_records(filter_dict, key, new_value) + + def update_many_records_custom(self, filter_dict, update_dict): + # Updates many records custom using update_dict + return self._writer.update_many_records_custom(filter_dict, update_dict) + """ + + def replace_record(self, filter_dict, new_entry): + """Deletes and replaces selected record""" + return self._writer.replace_record(filter_dict, new_entry) + + def delete_record_by_key(self, key, value): + """Deletes single record by key and matching value""" + return self._writer.delete_record_by_key(key, value) + + def update_library_card(self, library_name, update_dict, lib_card, delete_record=False): + + """Special update method to handle library card updates""" + + return self._writer.update_library_card(library_name, update_dict, lib_card, delete_record=delete_record) + + def get_and_increment_doc_id(self, library_name): + + """Gets and increments doc_id""" + + return self._writer.get_and_increment_doc_id(library_name) + + def set_incremental_docs_blocks_images(self, library_name, added_docs=0, added_blocks=0, added_images=0, + added_pages=0, added_tables=0): + + """Updates counts on library card""" + + return self._writer.set_incremental_docs_blocks_images(library_name, added_docs=added_docs, + added_blocks=added_blocks, + added_images=added_images, added_pages=added_pages, + added_tables=added_tables) + + def add_new_embedding_flag(self, _id, embedding_key, value): + """Updates JSON column of one record by adding new key:value""" + return self._writer.add_new_embedding_flag(_id, embedding_key,value) + + def unset_embedding_flag(self, embedding_key): + return self._writer.unset_embedding_flag(embedding_key) + + def close(self): + """Close connection to underlying DB resource""" + return self._writer.close() + + +class MongoWriter: + + """MongoWriter is main class abstraction for writes, edits and deletes to a Mongo text index collection""" + + def __init__(self, library_name, account_name="llmware", custom_table=False, custom_schema=None): + + self.library_name = library_name + self.account_name = account_name + self.uri_string = LLMWareConfig.get_db_uri_string() + + # initiate connection to Mongo resource + self.collection = _MongoConnect().connect(db_name=account_name, collection_name=library_name) + + self.custom_table = custom_table + self.custom_schema = custom_schema + + def build_text_index(self): + """Builds Mongo text search index""" + self.collection.create_index([("text_search", "text")]) + return True + + def check_if_table_build_required(self): + """Always returns False, since no table build steps required for Mongo no-sql DB""" + return False + + def create_table(self, table_name, schema): + """No table creation steps required in Mongo DB""" + return True + + def write_new_record(self, new_record): + """Inserts one new record in Mongo collection""" + + if "_id" in new_record: + new_record.update({"_id": ObjectId(new_record["_id"])}) + + registry_id = self.collection.insert_one(new_record).inserted_id + + return 1 + + def write_new_parsing_record(self, new_record): + """ Writes new parsing record into Mongo DB """ + return self.write_new_record(new_record) + + def destroy_collection(self, confirm_destroy=False): + + """Drops collection for library""" + if confirm_destroy: + self.collection.drop() + return 1 + + logger.warning("update: library not destroyed - need to set confirm_destroy = True") + return 0 + + def update_block (self, doc_id, block_id, key, new_value, default_keys): + + """Selects specific (doc_id, block_id) and updates with {key:new_value}""" + + completed = False + + f = {"$and": [{"block_ID": block_id}, {"doc_ID": doc_id}]} + + if key in default_keys: + new_values = {"$set": {key: new_value}} + self.collection.update_one(f,new_values) + completed = True + + return completed + + def update_one_record(self, filter_dict, key,new_value): + + """Updates one record selected by filter_dict, with {key:new_value}""" + + if "_id" in filter_dict: + filter_dict.update({"_id": ObjectId(filter_dict["_id"])}) + + new_values = {"$set": {key:new_value}} + self.collection.update_one(filter_dict, new_values) + return 0 + + """ + def update_many_records(self, filter_dict, key, new_value): + + # Updates many records selected by filter_dict, with {key:new_value} + + if "_id" in filter_dict: + filter_dict.update({"_id": ObjectId(filter_dict["_id"])}) + + new_values = {"$set": {key :new_value}} + self.collection.update_many(filter_dict, new_values) + return 0 + """ + """ + def update_many_records_custom(self, filter_dict, update_dict): + + # Updates many records using custom filter dict and potentially multiple updates + + if "_id" in filter_dict: + filter_dict.update({"_id": ObjectId(filter_dict["_id"])}) + + self.collection.update_many(filter_dict, update_dict) + return 0 + """ + + def replace_record(self, filter_dict, new_entry): + + """Replaces record in MongoDB collection""" + + if "_id" in filter_dict: + filter_dict.update({"_id": ObjectId(filter_dict["_id"])}) + + self.collection.replace_one(filter_dict, new_entry, upsert=True) + + return 1 + + def delete_record_by_key(self,key,value): + + """Deletes record by key matching value""" + + if key == "_id": + value = ObjectId(value) + + self.collection.delete_one({key:value}) + + return 1 + + def update_library_card(self, library_name, update_dict,lib_card, delete_record=False): + + """Updates library card in Mongo Library Catalog""" + + f = {"library_name": library_name} + new_values = {"$set": update_dict} + + embedding_list = lib_card["embedding"] + + # standard collection update for all except embedding + if "embedding" not in update_dict: + self.collection.update_one(f,new_values) + + else: + # special flag to identify where to 'merge' and update an existing embedding record + merged_embedding_update = False + inserted_list = [] + + if len(embedding_list) > 0: + # if the last row is a "no" embedding, then remove it + if embedding_list[-1]["embedding_status"] == "no": + del embedding_list[-1] + + for emb_records in embedding_list: + + if emb_records["embedding_model"] == update_dict["embedding"]["embedding_model"] and \ + emb_records["embedding_db"] == update_dict["embedding"]["embedding_db"]: + + if not delete_record: + inserted_list.append(update_dict["embedding"]) + else: + pass + + merged_embedding_update = True + + # catch potential error + + if not delete_record: + if "embedded_blocks" in emb_records and "embedded_blocks" in update_dict["embedding"]: + + if emb_records["embedded_blocks"] > update_dict["embedding"]["embedded_blocks"]: + + logger.warning(f"warning: may be issue with embedding - mis-alignment in " + f"embedding block count - " + f"{emb_records['embedded_blocks']} > " + f"{update_dict['embedding']['embedded_blocks']}") + + else: + inserted_list.append(emb_records) + + if not merged_embedding_update: + embedding_list.append(update_dict["embedding"]) + embedding_update_dict = {"embedding": embedding_list} + else: + embedding_update_dict = {"embedding": inserted_list} + + self.collection.update_one(f, {"$set": embedding_update_dict}) + + return 1 + + def get_and_increment_doc_id(self, library_name): + + """method called at the start of parsing each new doc -> each parser gets a new doc_id""" + + library_counts = self.collection.find_one_and_update( + {"library_name": library_name}, + {"$inc": {"unique_doc_id": 1}}, + return_document=ReturnDocument.AFTER + ) + + unique_doc_id = library_counts.get("unique_doc_id",-1) + + return unique_doc_id + + def set_incremental_docs_blocks_images(self, library_name, added_docs=0, added_blocks=0, added_images=0, + added_pages=0, added_tables=0): + + """updates counting parameters at end of parsing""" + + self.collection.update_one( + {"library_name": library_name}, + {"$inc": {"documents": added_docs, "blocks": added_blocks, "images": added_images, "pages": added_pages, + "tables": added_tables + }}) + + return 0 + + def add_new_embedding_flag(self,_id, embedding_key, value): + + filter_dict = {"_id": _id} + self.update_one_record (filter_dict, embedding_key, value) + + return 0 + + def unset_embedding_flag(self, embedding_key): + + update = {"$unset": {embedding_key: ""}} + self.collection.update_many({}, update) + + return 0 + + def close(self): + """Closes MongoDB connection""" + # self.collection.close() + return 0 + + +class MongoRetrieval: + + """MongoRetrieval is primary class abstraction to handle queries to Mongo text collection""" + + def __init__(self, library_name, account_name="llmware", custom_table=False, custom_schema=None): + + self.library_name = library_name + self.account_name = account_name + self.uri_string = LLMWareConfig.get_db_uri_string() + + # establish connection at construction of retrieval object + self.collection = _MongoConnect().connect(self.account_name,collection_name=self.library_name) + + self.reserved_tables = ["status", "library", "parser_events"] + self.text_retrieval = False + + if library_name not in self.reserved_tables: + self.text_retrieval = True + + self.custom_table = custom_table + if custom_table: + self.text_retrieval = False + + def safe_name(self, input_name): + + """ Mongo is flexible on collection names - for now, only filter is reserved collection names""" + + if input_name not in self.reserved_tables: + output_name = input_name + else: + raise LLMWareException(message=f"MongoRetrieval - selected name is not " + f"valid on database - {input_name}") + + return output_name + + def test_connection(self,timeout_secs=5): + + """Tests and confirms if connected to MongoDB""" + + client = MongoClient(self.uri_string, unicode_decode_error_handler='ignore') + + # self.client.admin.authenticate(username, password) + + try: + # catch if mongo not available + with pymongo.timeout(timeout_secs): + client.admin.command('ping') + logger.info(f"update: mongo connected - collection db available at uri string - " + f"{self.uri_string}") + db_status = True + + except ConnectionFailure: + logger.warning(f"warning: collection db not found at uri string in LLMWareConfig - " + f"{self.uri_string} - check connection and/or reset LLMWareConfig 'collection_db_uri' " + f"to point to the correct uri.") + + db_status = False + + return db_status + + def unpack(self, entry): + + """Unpack converts array row output to dictionary using schema, e.g., Identity function for MongoDB """ + + output = entry + + if isinstance(entry, list): + if len(entry) > 0: + if isinstance(entry[0], dict): + output = entry[0] + + return output + + def lookup(self, key, value): + + """Returns list of dictionary entries representing results""" + + # special handling for reserved id in Mongo + if key == "_id": + + try: + value = ObjectId(value) + except: + logger.debug(f"update: mongo lookup - could not find _id into ObjectID - {value}") + value = value + + target = list(self.collection.find({key:value})) + + return target + + def embedding_key_lookup(self, key, value): + return self.lookup(key,value) + + def get_whole_collection(self): + + """Retrieves whole collection in Mongo- will return as a Cursor object""" + + # update: removing no_cursor_timeout=True + # setting timeout to 30 minutes = 1,800,000 milliseconds + all_output = self.collection.find({}).max_time_ms(1800000) + + cursor = DBCursor(all_output,self, "mongo") + + return cursor + + def basic_query(self, query): + + """Basic text index query in MongoDB""" + + match_results_cursor = self.collection.find( + {"$text": {"$search": query}}, + {"score": {"$meta": "textScore"}}).sort([('score', {'$meta': 'textScore'})]).allow_disk_use(True) + + return match_results_cursor + + def filter_by_key(self, key, value): + + """Returns a cursor of entries in which key matches value""" + + match_results_cursor = list(self.collection.find({key:value})) + return match_results_cursor + + def text_search_with_key_low_high_range(self, query, key, low, high, key_value_dict=None): + + """Accepts key with low & high value + optional key_value_dict with additional parameters""" + + d = [] + f = {} + + text_search = {"$text": {"$search": query}} + d.append(text_search) + key_value_range = {key: {"$gte": low, "$lte": high}} + d.append(key_value_range) + + if key_value_dict: + for key, value in key_value_dict.items(): + d.append({key: value}) + + # if one key-value pair, then simple filter + if len(d) == 1: f = d[0] + + # if multiple key-value pairs, then insert list with "$and" + if len(d) >= 2: + f = {"$and": d} + + results = list(self.collection.find(f, + {"score": {"$meta": "textScore"}}). + sort([('score', {'$meta': 'textScore'})]).allow_disk_use(True)) + + return results + + def text_search_with_key_value_range(self, query, key, value_range_list, key_value_dict=None): + + """Text search with additional constraint of key in provided value_range list""" + + f = {} + text_search = {"$text": {"$search": query}} + + d = [text_search] + range_filter = {key: {"$in": value_range_list}} + d.append(range_filter) + + if key_value_dict: + for key, value in key_value_dict.items(): + d.append({key: value}) + + # if one key-value pair, then simple filter + if len(d) == 1: f = d[0] + + # if multiple key-value pairs, then insert list with "$and" + if len(d) >= 2: + f = {"$and": d} + + results = list(self.collection.find(f, + {"score": {"$meta": "textScore"}}). + sort([('score', {'$meta': 'textScore'})]).allow_disk_use(True)) + + return results + + def text_search_with_key_value_dict_filter(self, query, key_value_dict): + + """Text search with additional key_value filter dictionary applied""" + + f = {} + text_search = {"$text": {"$search": query}} + d = [text_search] + for key, value in key_value_dict.items(): + + if isinstance(value, list): + # if value is a list, then interpret as "$in" + range_filter = {key: {"$in": value}} + d.append(range_filter) + else: + # if value is not a list, then look for exact match + d.append({key: value}) + + # if one key-value pair, then simple filter + if len(d) == 1: f = d[0] + + # if multiple key-value pairs, then insert list with "$and" + if len(d) >= 2: + f = {"$and": d} + + results = list(self.collection.find(f, + {"score": {"$meta": "textScore"}}). + sort([('score', {'$meta': 'textScore'})]).allow_disk_use(True)) + + return results + + def get_distinct_list(self, key): + + """Returns distinct list of items by key""" + # not using distinct operation + # distinct can break due to the number of entries in the library + # to prevent this from happen we use a aggregate which does not produce a document but a cursor + # we loop the cursor and so we overcome the distinct operation 16mb document cap + + group = self.collection.aggregate([{ "$group": {"_id": f'${key}',}}]) + + distinct_list = [] + for entry in group: + distinct_list.append(entry['_id']) + + return distinct_list + + def filter_by_key_dict (self, key_dict): + + """Filters collection by key-value dictionary""" + + f = {} + d = [] + for key, value in key_dict.items(): + d.append({key :value}) + + # if one key-value pair, then simple filter + if len(d) == 1: f = d[0] + + # if multiple key-value pairs, then insert list with "$and" + if len(d) >= 2: f= {"$and":d} + + results = list(self.collection.find(f)) + + return results + + def filter_by_key_value_range(self, key, value_range): + + """Filter by key matching value_range list, e.g., {"doc_ID": [1,2,3,4,5]}""" + + results = list(self.collection.find({key: {"$in": value_range}})) + return results + + def filter_by_key_ne_value(self, key, value): + + """Filter by key not equal to specific value""" + + f = {key: {"$ne":value}} + output = list(self.collection.find(f)) + return output + + def count_documents(self, filter_dict): + + """Count documents that match filter conditions""" + + num_of_blocks = self.collection.count_documents(filter_dict) + return num_of_blocks + + def embedding_job_cursor(self, new_embedding_key,doc_id=None): + + """Handles end-to-end retrieval of text blocks selected for embedding - returns Cursor""" + + if doc_id: + filter_dict = {"doc_ID":{"$in": doc_id}} + num_of_blocks = self.count_documents(filter_dict) + all_blocks_cursor = self.collection.find(filter_dict) + + else: + filter_dict = {new_embedding_key: {"$exists": False}} + num_of_blocks = self.count_documents(filter_dict) + all_blocks_cursor = self.collection.find(filter_dict) + + cursor = DBCursor(all_blocks_cursor,self, "mongo") + + return num_of_blocks, cursor + + def count_embedded_blocks(self, embedding_key): + + """Counts number of text blocks to be embedded in current embedding job scope""" + + filter_dict = {embedding_key: {"$exists": True}} + embedded_blocks = self.count_documents(filter_dict) + + return embedded_blocks + + def close(self): + """Closing MongoDB connection not required - no action taken""" + # self.collection.close() + return 0 + + def direct_custom_query(self, query_filter): + + """Applies the custom query directly to the DB and returns the results""" + + # will force exhausting cursor iterable into list - designed for relatively small in-memory retrievals + results = list(self.collection.find(query_filter)) + + return results + + def list_all_tables(self): + + """Get list of all collections on the database""" + # self.mongo_client = MongoDBManager().client[db_name][collection_name] + + results = list(MongoDBManager().client[self.account_name].list_collection_names()) + + return results + + def get_schema(self, table_name): + + """ No schema in Mongo collections so returns empty value """ + + table_schema = {} + + return table_schema + + +class PGRetrieval: + + """PGRetrieval is main class to handle interactions with Postgres DB for queries and retrieval - + Embedding connections through PGVector are handled separately through EmbeddingPGVector class""" + + def __init__(self, library_name, account_name="llmware", custom_table=False, custom_schema=None): + + self.account_name = account_name + self.library_name = library_name + + self.conn = _PGConnect().connect() + + self.reserved_tables = ["status", "library", "parser_events"] + self.text_retrieval = False + + if library_name == "status": + self.schema = LLMWareTableSchema().get_status_schema() + elif library_name == "library": + self.schema = LLMWareTableSchema().get_library_card_schema() + elif library_name == "parser_events": + self.schema = LLMWareTableSchema().get_parser_table_schema() + else: + self.schema = LLMWareTableSchema().get_block_schema() + + if library_name not in self.reserved_tables: + self.text_retrieval = True + + self.custom_table=custom_table + if custom_table: + self.text_retrieval = False + + if custom_schema: + self.schema = custom_schema + + def test_connection(self): + + """Test connection to Postgres database""" + test = True + + try: + # try to open and close connection + test_connection = _PGConnect().connect() + test_connection.close() + except: + # if error, then catch and fail test + test = False + + return test + + def safe_name(self, input_name): + + """ Table names in Postgres must consist of alpha, numbers and _ -> does not permit '-' """ + + if input_name not in self.reserved_tables: + output_name = re.sub("-","_", input_name) + else: + raise LLMWareException(message=f"PGRetrieval - selected name is not " + f"valid on database - {input_name}") + + return output_name + + def unpack(self, results_cursor): + + """Iterate through rows of results_cursor and builds dictionary output rows using schema""" + + output = [] + + for row in results_cursor: + + counter = 0 + new_dict = {} + + for key, value in self.schema.items(): + + if key != "PRIMARY KEY": + if counter < len(row): + if key == "text_block": + key = "text" + if key == "table_block": + key = "table" + new_dict.update({key: row[counter]}) + counter += 1 + else: + logger.warning(f"update: pg_retriever - outputs not matching - {counter}") + + output.append(new_dict) + + return output + + def unpack_search_result(self, results_cursor): + + """Iterate through rows of results_cursor and builds dictionary output rows using schema""" + + output = [] + + for row in results_cursor: + + counter = 0 + new_dict = {} + new_dict.update({"score": row[0]}) + counter += 1 + + for key, value in self.schema.items(): + + if key != "PRIMARY KEY": + if counter < len(row): + if key == "text_block": + key = "text" + if key == "table_block": + key = "table" + new_dict.update({key: row[counter]}) + counter += 1 + else: + logger.warning (f"update: pg_retriever - outputs not matching - {counter}") + + output.append(new_dict) + + return output + + def lookup(self, key, value): + + """Lookup returns entry with key (column) with matching value - returns as unpacked dict entry""" + + output = {} + + sql_query = f"SELECT * FROM {self.library_name} WHERE {key} = '{value}';" + results = list(self.conn.cursor().execute(sql_query)) + + if results: + if len(results) >= 1: + output = self.unpack(results) + + self.conn.close() + + return output + + def embedding_key_lookup(self, key, value): + + # lookup in json dictionary - special sql command + output = [] + value = str(value) + + sql_query= f"SELECT * FROM {self.library_name} WHERE embedding_flags->>'{key}' = '{value}'" + + results = list(self.conn.cursor().execute(sql_query)) + + if results: + if len(results) >= 1: + output = self.unpack(results) + + self.conn.close() + + return output + + def get_whole_collection(self): + + """Returns whole collection - as a Cursor object""" + + sql_command = f"SELECT * FROM {self.library_name}" + results = self.conn.cursor().execute(sql_command) + + cursor = DBCursor(results,self, "postgres") + + # self.conn.close() + + return cursor + + def _prep_query(self, query): + + """ Simple query text preparation - will add more options over time """ + + pg_strings = {"AND": " & ", "OR": " | "} + + exact_match = False + # check if wrapped in quotes + if query.startswith('"') and query.endswith('"'): + exact_match = True + + # remove punctuation and split into tokens by whitespace + q_clean = re.sub(r"[^\w\s]", "", query) + q_toks = q_clean.split(" ") + + q_string = "" + for tok in q_toks: + q_string += tok + if exact_match: + # q_string += " & " + q_string += pg_strings["AND"] + else: + # q_string += " | " + q_string += pg_strings["OR"] + + if q_string.endswith(pg_strings["AND"]): + q_string = q_string[: -len(pg_strings["AND"])] + + if q_string.endswith(pg_strings["OR"]): + # if q_string.endswith(" & ") or q_string.endswith(" | "): + q_string = q_string[:-len(pg_strings["OR"])] + + return q_string + + def basic_query(self, query): + + """Basic Postgres tsquery text query""" + + search_string = self._prep_query(query) + + sql_query = f"SELECT ts_rank_cd (ts, to_tsquery('english', '{search_string}')) as rank, * " \ + f"FROM {self.library_name} " \ + f"WHERE ts @@ to_tsquery('english', '{search_string}') " \ + f"ORDER BY rank DESC LIMIT 100 ;" + + results = self.conn.cursor().execute(sql_query) + + output_results = self.unpack_search_result(results) + + self.conn.close() + + return output_results + + def filter_by_key(self, key, value): + + """SELECT ... WHERE {key} = '{value}'""" + + output = [{}] + + sql_query = f"SELECT * FROM {self.library_name} WHERE {key} = '{value}';" + results = self.conn.cursor().execute(sql_query) + + if results: + output = self.unpack(results) + + self.conn.close() + + return output + + def text_search_with_key_low_high_range(self, query, key, low, high, key_value_dict=None): + + """Text search with additional constraint of matching column with value in specified range""" + + search_string = self._prep_query(query) + + sql_query = f"SELECT ts_rank_cd (ts, to_tsquery('english', '{search_string}')) as rank, * " \ + f"FROM {self.library_name} " \ + f"WHERE ts @@ to_tsquery('english', '{search_string}') " \ + f"AND {key} BETWEEN {low} AND {high}" + + if key_value_dict: + for key, value in key_value_dict.items(): + sql_query += f" AND {key} = {value}" + + sql_query += " ORDER by rank" + sql_query += ";" + + results = self.conn.cursor().execute(sql_query) + + output_results = self.unpack_search_result(results) + + self.conn.close() + + return output_results + + def text_search_with_key_value_range(self, query, key, value_range_list, key_value_dict=None): + + """Text search with additional constraint(s) of keys matching values in value_range list and + optional key_value_dict""" + + search_string = self._prep_query(query) + + ia_str = "(" + for v in value_range_list: + if isinstance(v, int): + ia_str += str(v) + else: + ia_str += "'" + v + "'" + ia_str += ", " + if ia_str.endswith(", "): + ia_str = ia_str[:-2] + ia_str += ")" + + # ia_str = "(1)" + + sql_query = f"SELECT ts_rank_cd (ts, to_tsquery('english', '{search_string}')) as rank, * " \ + f"FROM {self.library_name} " \ + f"WHERE ts @@ to_tsquery('english', '{search_string}') " \ + f"AND {key} IN {ia_str}" + + if key_value_dict: + for key, value in key_value_dict.items(): + sql_query += f" AND {key} = {value}" + + sql_query += " ORDER BY rank" + sql_query += ";" + + results = self.conn.cursor().execute(sql_query) + + output_results = self.unpack_search_result(results) + + self.conn.close() + + return output_results + + def text_search_with_key_value_dict_filter(self, query, key_value_dict): + + """Text search with additional "AND" constraints of key value dict with key = value""" + + search_string = self._prep_query(query) + + sql_query = f"SELECT ts_rank_cd (ts, to_tsquery('english', '{search_string}')) as rank, * " \ + f"FROM {self.library_name} " \ + f"WHERE ts @@ to_tsquery('english', {search_string})" + + if key_value_dict: + for key, value in key_value_dict.items(): + + if isinstance(value,list): + + # need to check this + value_range = str(value) + value_range = value_range.replace("[", "(") + value_range = value_range.replace("]", ")") + + sql_query += f" AND {key} IN {value_range}" + else: + sql_query += f" AND {key} = '{value}'" + + sql_query += " ORDER BY rank" + sql_query += ";" + + results = self.conn.cursor().execute(sql_query) + output_results = self.unpack_search_result(results) + + self.conn.close() + + return output_results + + def get_distinct_list(self, key): + + """Returns distinct list by col (key)""" + + sql_query = f"SELECT DISTINCT {key} FROM {self.library_name};" + results = self.conn.cursor().execute(sql_query) + + output = [] + for res in results: + if res: + if len(res) > 0: + output.append(res[0]) + + self.conn.close() + + return output + + def filter_by_key_dict (self, key_dict): + + """Returns rows selected by where conditions set forth in key-value dictionary""" + + sql_query = f"SELECT * FROM {self.library_name}" + + conditions_clause = " WHERE" + for key, value in key_dict.items(): + + # handles passing a filter with 'mongo' style $in key range + if isinstance(value,dict): + if "$in" in value: + value = value["$in"] + + logger.debug(f"update: Postgres - filter_by_key_dict - value - {value}") + + if isinstance(value,list): + v_str = "(" + for entry in value: + v_str += str(entry) + "," + if v_str.endswith(","): + v_str = v_str[:-1] + v_str += ")" + conditions_clause += f" {key} IN {v_str} AND " + else: + + conditions_clause += f" {key} = '{value}' AND " + + if conditions_clause.endswith(' AND '): + conditions_clause = conditions_clause[:-5] + if len(conditions_clause) > len(" WHERE"): + sql_query += conditions_clause + ";" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def filter_by_key_value_range(self, key, value_range): + + """Filter by key in value range, e.g., {"doc_ID": [1,2,3,4,5]}""" + + value_range_str = "(" + for v in value_range: + value_range_str += "'" + str(v) + "'" + ", " + if value_range_str.endswith(", "): + value_range_str = value_range_str[:-2] + value_range_str += ")" + + sql_query = f"SELECT * from {self.library_name} WHERE {key} IN {value_range_str};" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def filter_by_key_ne_value(self, key, value): + + """Filter by col (key) not equal to specified value""" + + sql_query = f"SELECT * from {self.library_name} WHERE NOT {key} = {value};" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def embedding_job_cursor(self, new_embedding_key, doc_id=None): + + """Handles end-to-end retrieval of text blocks selected for embedding job - returns Cursor""" + + if doc_id: + + # pull selected documents for embedding + insert_array = () + insert_array += (tuple(doc_id),) + + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE doc_ID IN %s;" + count_result = list(self.conn.cursor().execute(sql_query, insert_array)) + count = count_result[0] + + sql_query = f"SELECT * FROM {self.library_name} WHERE doc_ID IN %s;" + results = self.conn.cursor().execute(sql_query, insert_array) + + else: + + # first get the total count of blocks 'un-embedded' with this key in the collection + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE embedding_flags->>'{new_embedding_key}' " \ + f"is NULL;" + + count_result = list(self.conn.cursor().execute(sql_query)) + count = count_result[0] + + sql_query = f"SELECT * FROM {self.library_name} WHERE embedding_flags->>'{new_embedding_key}' is NULL;" + results = self.conn.cursor().execute(sql_query) + + cursor = DBCursor(results,self,"postgres") + + return count[0], cursor + + def count_embedded_blocks(self, embedding_key): + + """Counts the total number of blocks to be embedded in current job scope""" + + # send error code by default if can not count from db directly + embedded_blocks = -1 + + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE embedding_flags->>'{embedding_key}' is NOT NULL;" + results = list(self.conn.cursor().execute(sql_query)) + + if len(results) > 0: + embedded_blocks = results[0] + + if not isinstance(embedded_blocks, int): + if len(embedded_blocks) > 0: + embedded_blocks = embedded_blocks[0] + + self.conn.close() + + return embedded_blocks + + def count_documents(self, filter_dict): + + """Count documents that match filter conditions""" + conditions_clause = "" + + if filter_dict: + + for key, value in filter_dict.items(): + conditions_clause += f"{key} = {value} AND " + + if conditions_clause.endswith(" AND "): + conditions_clause = conditions_clause[:-5] + + if conditions_clause: + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE {conditions_clause};" + else: + sql_query = f"SELECT COUNT(*) FROM {self.library_name};" + + results = list(self.conn.cursor().execute(sql_query)) + + output = results[0] + + self.conn.close() + + return output + + def close(self): + + """Closes Postgres connection""" + + self.conn.close() + return 0 + + def direct_custom_query(self, query_filter): + + """Applies the custom query directly to the DB and returns the results""" + results = list(self.conn.cursor().execute(str(query_filter))) + # output = self.unpack(results) + self.conn.close() + return results + + def list_all_tables(self): + + """Get list of all tables on the database""" + + sql_query = f"SELECT * FROM pg_tables;" + + list_of_tables = list(self.conn.cursor().execute(sql_query)) + + self.conn.close() + + return list_of_tables + + def get_schema(self, table_name): + + sql_query = (f"SELECT ordinal_position, column_name, data_type " + f"FROM information_schema.columns " + f"WHERE table_schema = 'public' " + f"AND table_name = '{table_name}';") + + # sql_query = f"SELECT * FROM pg_tables WHERE tablename = '{table_name}';" + table_results = list(self.conn.cursor().execute(sql_query)) + self.conn.close() + + # unpack into a dictionary from list + table_results_sorted = sorted(table_results, key=lambda x:x[0], reverse=False) + schema_dict = {} + for i, entry in enumerate(table_results_sorted): + if len(entry) >= 3 and (entry[0]==i+1): + if entry[1] not in schema_dict: + schema_dict.update({entry[1]:entry[2]}) + + return schema_dict + + +class PGWriter: + + """PGWriter is main class abstraction to handle writing, indexing, modifying and deleting records in + Postgres tables""" + + def __init__(self, library_name, account_name="llmware", custom_table=False, custom_schema=None): + + self.library_name = library_name + self.account_name = account_name + + self.conn = _PGConnect().connect() + + # simple lookup of schema by supported table type + + if library_name == "status": + self.schema = LLMWareTableSchema().get_status_schema() + + elif library_name == "library": + self.schema = LLMWareTableSchema().get_library_card_schema() + + elif library_name == "parser_events": + self.schema = LLMWareTableSchema().get_parser_table_schema() + + else: + # default is to assign as a 'block' text collection schema + self.schema = LLMWareTableSchema().get_block_schema() + + self.reserved_tables = ["status", "library", "parser_events"] + self.text_retrieval = False + + if library_name not in self.reserved_tables: + self.text_retrieval = True + + self.custom_table = custom_table + + if custom_table: + self.text_retrieval = False + if custom_schema: + self.schema = custom_schema + + def _add_search_column(self, search_col="ts"): + + """Creates ts_vector search column = ts to enable text_search on Postgres DB""" + + sql_add_ts_col = f"ALTER TABLE {self.library_name} ADD COLUMN {search_col} tsvector " \ + f"GENERATED ALWAYS AS(to_tsvector('english', text_search)) STORED;" + + self.conn.execute(sql_add_ts_col) + + self.conn.commit() + + return 0 + + def build_text_index(self): + + """Creates GIN index on new search column to enable text index search on Postgres DB""" + + sql_text_index_create = f"CREATE INDEX IF NOT EXISTS ts_idx ON {self.library_name} USING GIN(ts);" + + self.conn.execute(sql_text_index_create) + self.conn.commit() + self.conn.close() + + return 1 + + def check_if_table_build_required(self): + + """Check if table already exists""" + + build_table = True + table_name = self.library_name + + sql_query = f"SELECT * FROM pg_tables WHERE tablename = '{table_name}';" + + test_result = list(self.conn.cursor().execute(sql_query)) + + if len(test_result) > 0: + if table_name in test_result[0]: + build_table = False + + # self.conn.close() + + return build_table + + def _build_sql_from_schema (self, table_name, schema): + + """Utility function to build sql from a schema dictionary""" + + table_create = f"CREATE TABLE IF NOT EXISTS {table_name} (" + + for key, value in schema.items(): + table_create += key + " " + value + ", " + + if table_create.endswith(", "): + table_create = table_create[:-2] + + table_create += ");" + + return table_create + + def create_table(self, table_name, schema, add_search_column=True): + + """ Creates table with selected name and schema""" + + # only add search index to library blocks collection using self.library_name + + if table_name in ["status", "library", "parser_events"] or self.custom_table: + add_search_column = False + else: + add_search_column = True + + table_create = self._build_sql_from_schema(table_name, schema) + + self.conn.execute(table_create) + + if add_search_column: + self._add_search_column() + + self.conn.commit() + + # close connection at end of update + self.conn.close() + + return 1 + + def write_new_record(self, new_record): + + """Writes new record - primary for creating new library card and status update""" + + keys_list = "(" + output_values = "(" + + for keys, values in new_record.items(): + + keys_list += keys + ", " + + new_entry = str(new_record[keys]) + new_entry = new_entry.replace("'", '"') + + output_values += "'" + new_entry + "'" + ", " + + if keys_list.endswith(", "): + keys_list = keys_list[:-2] + + if output_values.endswith(", "): + output_values = output_values[:-2] + + keys_list += ")" + output_values += ")" + + sql_instruction = f"INSERT INTO {self.library_name} {keys_list} VALUES {output_values};" + + results = self.conn.cursor().execute(sql_instruction) + + self.conn.commit() + + self.conn.close() + + return 1 + + def write_new_parsing_record(self, rec): + + """ Writes new parsing record dictionary into Postgres """ + + sql_string = f"INSERT INTO {self.library_name}" + sql_string += " (block_ID, doc_ID, content_type, file_type, master_index, master_index2, " \ + "coords_x, coords_y, coords_cx, coords_cy, author_or_speaker, added_to_collection, " \ + "file_source, table_block, modified_date, created_date, creator_tool, external_files, " \ + "text_block, header_text, text_search, user_tags, special_field1, special_field2, " \ + "special_field3, graph_status, dialog, embedding_flags) " + + sql_string += " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \ + "%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);" + + # now unpack the new_record into parameters + insert_arr = (rec["block_ID"], rec["doc_ID"],rec["content_type"], rec["file_type"], rec["master_index"], + rec["master_index2"], rec["coords_x"], rec["coords_y"], rec["coords_cx"], rec["coords_cy"], + rec["author_or_speaker"], rec["added_to_collection"], rec["file_source"], rec["table"], + rec["modified_date"], rec["created_date"], rec["creator_tool"], rec["external_files"], + rec["text"], rec["header_text"], rec["text_search"], rec["user_tags"], + rec["special_field1"], rec["special_field2"], rec["special_field3"], rec["graph_status"], + rec["dialog"], str(rec["embedding_flags"])) + + # note: sets embedding_flag value (last parameter) to "{}" = str({}) + + results = self.conn.cursor().execute(sql_string,insert_arr) + + self.conn.commit() + + self.conn.close() + + return True + + def destroy_collection(self, confirm_destroy=False): + + """Drops table from database""" + + if confirm_destroy: + + sql_instruction = f"DROP TABLE {self.library_name};" + + # returns TRUE if table does not exist & FALSE if table exists + table_does_not_exist = self.check_if_table_build_required() + + # if FALSE ... drop the table + if not table_does_not_exist: + results = self.conn.cursor().execute(sql_instruction) + self.conn.commit() + self.conn.close() + return 1 + else: + logger.warning(f"update: PGWriter - request to drop table not executed because table " + f"could not be found in the database.") + return -1 + + logger.warning("update: library not destroyed - need to set confirm_destroy = True") + + # self.conn.commit() + self.conn.close() + + return 0 + + def update_block (self, doc_id, block_id, key, new_value, default_keys): + + """Lookup block by doc_id & block_id and update with specific key and new value""" + + completed = False + + if key in default_keys: + + sql_instruction = f"UPDATE {self.library_name} "\ + f"SET {key} = {new_value} " \ + f"WHERE doc_ID = {doc_id} AND block_ID = {block_id};" + + completed = True + results = self.conn.cursor().execute(sql_instruction) + + self.conn.commit() + + self.conn.close() + + return completed + + def update_one_record(self, filter_dict, key, new_value): + + """Updates one record""" + + conditions_clause = "" + for k, v in filter_dict.items(): + conditions_clause += f"{k} = '{v}' AND" + + if conditions_clause.endswith(" AND"): + conditions_clause = conditions_clause[:-4] + + if conditions_clause: + sql_instruction = f"UPDATE {self.library_name} " \ + f"SET {key} = '{new_value}' " \ + f"WHERE {conditions_clause};" + + results = self.conn.cursor().execute(sql_instruction) + self.conn.commit() + + self.conn.close() + + return 0 + + def replace_record(self, filter_dict, new_entry): + + """Check if existing record with the same key - if so, delete, then create new""" + + new_values = "(" + for keys, values in new_entry.items(): + new_values += "'" + str(values) + "', " + if new_values.endswith(", "): + new_values = new_values[:-2] + new_values += ")" + + conditions_clause = "" + for keys, values in filter_dict.items(): + conditions_clause += f"{keys} = '{values}' AND" + if conditions_clause.endswith(" AND"): + conditions_clause = conditions_clause[:-4] + + sql_check = f"SELECT * FROM {self.library_name} WHERE {conditions_clause};" + + exists = list(self.conn.cursor().execute(sql_check)) + + if exists: + # need to delete, then replace with new record + + sql_delete = f"DELETE FROM {self.library_name} WHERE {conditions_clause};" + self.conn.cursor().execute(sql_delete) + + sql_new_insert = f"INSERT INTO {self.library_name} VALUES {new_values};" + + self.conn.cursor().execute(sql_new_insert) + self.conn.commit() + + self.conn.close() + + return 0 + + def delete_record_by_key(self,key,value): + + """Deletes record found by matching key = value""" + + sql_command = f"DELETE FROM {self.library_name} WHERE {key} = '{value}';" + self.conn.execute(sql_command) + self.conn.commit() + self.conn.close() + return 0 + + def update_library_card(self, library_name, update_dict, lib_card, delete_record=False): + + """Updates library card""" + + conditions_clause = f"library_name = '{library_name}'" + + update_embedding_record = False + insert_array = () + + update_clause = "" + for key, new_value in update_dict.items(): + + if key != "embedding": + + if isinstance(new_value, int): + # update_clause += f"{key} = {new_value}, " + update_clause += f"{key} = %s" + insert_array += (new_value,) + else: + # update_clause += f"{key} = '{new_value}', " + update_clause += f"{key} = %s" + insert_array += (new_value,) + else: + # will update in second step + current_emb_record = lib_card["embedding"] + embedding_update = self._update_embedding_record_handler(current_emb_record, new_value, + delete_record=delete_record) + embedding_update = json.dumps(embedding_update) + # embedding_update = str(embedding_update).replace("'", '"') + # update_clause += f"{key} = '{embedding_update}', " + update_clause += f"{key} = %s, " + insert_array += (embedding_update,) + + if update_clause.endswith(", "): + update_clause = update_clause[:-2] + + sql_instruction = f"UPDATE {self.library_name} " \ + f"SET {update_clause} " \ + f"WHERE {conditions_clause};" + + self.conn.cursor().execute(sql_instruction, insert_array) + self.conn.commit() + + self.conn.close() + + return 1 + + def _update_embedding_record_handler(self, embedding_list, new_value,delete_record=False): + + """Internal helper method to integrate embedding update into array of dicts- which + is inserted as JSON directly in Postgres""" + + # special flag to identify where to 'merge' and update an existing embedding record + merged_embedding_update = False + inserted_list = [] + + if len(embedding_list) > 0: + # if the last row is a "no" embedding, then remove it + if embedding_list[-1]["embedding_status"] == "no": + del embedding_list[-1] + + for emb_records in embedding_list: + + if emb_records["embedding_model"] == new_value["embedding_model"] and \ + emb_records["embedding_db"] == new_value["embedding_db"]: + + if not delete_record: + inserted_list.append(new_value) + else: + pass + + merged_embedding_update = True + + # catch potential error + + if not delete_record: + if "embedded_blocks" in emb_records and "embedded_blocks" in new_value: + + if emb_records["embedded_blocks"] > new_value["embedded_blocks"]: + + logger.warning(f"warning: may be issue with embedding - mis-alignment in " + f"embedding block count - " + f"{emb_records['embedded_blocks']} > {new_value['embedded_blocks']}") + + else: + inserted_list.append(emb_records) + + if not merged_embedding_update: + embedding_list.append(new_value) + + output = embedding_list + else: + output = inserted_list + + return output + + def get_and_increment_doc_id (self, library_name): + + """Gets and increments unique doc ID""" + + val_out = -1 + + val_array = (str(library_name),) + + sql_instruction = f"UPDATE library " \ + f"SET unique_doc_id = unique_doc_id + 1 " \ + f"WHERE library_name = %s " \ + f"RETURNING unique_doc_id" + + result = self.conn.cursor().execute(sql_instruction, val_array) + + output = list(result) + if len(output) > 0: + val = output[0] + if len(val) > 0: + val_out = val[0] + + self.conn.commit() + self.conn.close() + + return val_out + + def set_incremental_docs_blocks_images(self, library_name, added_docs=0, added_blocks=0, added_images=0, + added_pages=0, added_tables=0): + + """Updates library card after update of new parsing jobs""" + + conditions_clause = f"library_name = '{library_name}'" + + set_clause = f"documents = documents + {added_docs}, " \ + f"blocks = blocks + {added_blocks}, " \ + f"images = images + {added_images}, " \ + f"pages = pages + {added_pages}, " \ + f"tables = tables + {added_tables}" + + sql_instruction = f"UPDATE {self.library_name} SET {set_clause} WHERE {conditions_clause};" + + results = self.conn.cursor().execute(sql_instruction) + + self.conn.commit() + + self.conn.close() + + return 0 + + def add_new_embedding_flag(self, _id, embedding_key, value): + + insert_array = () + + insert_json = f'X"{embedding_key}": "{value}"Y' + insert_json = insert_json.replace("X", "{") + insert_json = insert_json.replace("Y", "}") + insert_json = "'" + insert_json + "'" + insert_json += "::jsonb" + + json_dict = json.dumps({embedding_key:value}) + insert_array += (json_dict,) + + sql_command = f"UPDATE {self.library_name} " \ + f"SET embedding_flags = coalesce(embedding_flags, 'XY') || %s WHERE _id = {_id}" + sql_command = sql_command.replace("X","{") + sql_command = sql_command.replace("Y","}") + + self.conn.cursor().execute(sql_command, insert_array) + self.conn.commit() + self.conn.close() + + return 0 + + def unset_embedding_flag(self, embedding_key): + + """To complete deletion of an embedding, remove the json embedding_key from the text collection""" + + sql_instruction = f"UPDATE {self.library_name} " \ + f"SET embedding_flags = embedding_flags - {embedding_key}" \ + f"WHERE embedding_flags->>{embedding_key} IS NOT NULL" + + self.conn.cursor().execute(sql_instruction) + self.conn.commit() + self.conn.close() + + return 0 + + def close(self): + """Closes Postgres connection""" + self.conn.close() + return 0 + + +class SQLiteRetrieval: + + """SQLiteRetrieval is main class abstraction to handle queries and retrievals from a SQLite DB running locally""" + + def __init__(self, library_name, account_name="llmware", custom_table=False, custom_schema=None): + + self.library_name = library_name + self.account_name = account_name + + self.conn = _SQLiteConnect().connect(library_name) + + self.conn.text_factory = lambda b: b.decode(errors='ignore') + + self.reserved_tables = ["status", "library", "parser_events"] + self.text_retrieval = False + + if library_name == "status": + self.schema = LLMWareTableSchema().get_status_schema() + + elif library_name == "library": + self.schema = LLMWareTableSchema().get_library_card_schema() + + elif library_name == "parser_events": + self.schema = LLMWareTableSchema().get_parser_table_schema() + + else: + self.schema = LLMWareTableSchema().get_block_schema() + + if library_name not in self.reserved_tables: + self.text_retrieval = True + + self.custom_table = custom_table + if custom_table: + self.text_retrieval = False + + if custom_schema: + self.schema = custom_schema + + def test_connection(self): + + """SQLite test connection always returns True - runs in file system""" + + return True + + def safe_name(self, input_name): + + """ Conforming table name rules in Sqlite to Postgres """ + + if input_name not in self.reserved_tables: + output_name = re.sub("-", "_", input_name) + else: + raise LLMWareException(message=f"SQLiteRetrieval - selected name is not " + f"valid on database - {input_name}") + + return output_name + + def unpack(self, results_cursor): + + """Iterate through rows of results_cursor and builds dictionary output rows using schema""" + + output = [] + + for row in results_cursor: + + counter = 0 + new_dict = {} + + # assumes rowid included + new_dict.update({"_id": row[0]}) + counter += 1 + + for key, value in self.schema.items(): + + if key not in ["_id","PRIMARY KEY"]: + + if counter < len(row): + + output_value = row[counter] + + if key == "text_block": + key = "text" + + if key == "table_block": + key = "table" + + if key == "embedding": + output_value = ast.literal_eval(output_value) + + new_dict.update({key: output_value}) + counter += 1 + + else: + logger.warning(f"update: sqlite_retriever - outputs not matching -{counter}") + + output.append(new_dict) + + return output + + def unpack_search_result(self, results_cursor): + + """Iterate through rows of results_cursor and builds dictionary output rows using schema""" + + # assumes prepending of score + rowid + + output = [] + + for row in results_cursor: + + counter = 0 + new_dict = {} + new_dict.update({"score": row[0]}) + counter += 1 + new_dict.update({"_id": row[1]}) + counter += 1 + + for key, value in self.schema.items(): + + if key not in ["_id", "PRIMARY KEY"]: + + if counter < len(row): + + if key == "text_block": + key = "text" + + if key == "table_block": + key = "table" + + new_dict.update({key: row[counter]}) + counter += 1 + + else: + logger.warning(f"update: sqlite_retriever - outputs not matching - {counter}") + + output.append(new_dict) + + return output + + def lookup(self, key, value): + + """Lookup of col (key) matching to value - returns unpacked dictionary result""" + + output = {} + + if key == "_id": + key = "rowid" + + sql_query = f"SELECT rowid, * FROM {self.library_name} WHERE {key} = '{value}';" + + results = list(self.conn.cursor().execute(sql_query)) + + if results: + if len(results) >= 1: + output = self.unpack(results) + + self.conn.close() + + return output + + def embedding_key_lookup(self, key, value): + + output = {} + + value = str(value) + + # lookup embedding_flag = value and value in special_field1 + sql_command = (f"SELECT rowid, * FROM {self.library_name} WHERE embedding_flags = '{key}' AND " + f"special_field1 = '{value}'") + + results = list(self.conn.cursor().execute(sql_command)) + + if len(results) > 0: + output = self.unpack(results) + + return output + + def get_whole_collection(self): + + """Returns whole collection - as a Cursor object""" + + sql_command = f"SELECT rowid, * FROM {self.library_name}" + results = self.conn.cursor().execute(sql_command) + + cursor = DBCursor(results,self, "sqlite") + + return cursor + + def _prep_query(self, query): + + """ Basic preparation of text search query for SQLite - will evolve over time. """ + + sqlite_strings = {"AND": " AND ", "OR": " OR "} + + exact_match = False + # check if wrapped in quotes + if query.startswith('"') and query.endswith('"'): + exact_match = True + + # remove punctuation and split into tokens by whitespace + q_clean = re.sub(r"[^\w\s]", "", query) + q_toks = q_clean.split(" ") + + q_string = "" + for tok in q_toks: + q_string += tok + if exact_match: + # q_string += " & " + q_string += sqlite_strings["AND"] + else: + # q_string += " | " + q_string += sqlite_strings["OR"] + + if q_string.endswith(sqlite_strings["AND"]): + q_string = q_string[:-len(sqlite_strings["AND"])] + + if q_string.endswith(sqlite_strings["OR"]): + q_string = q_string[:-len(sqlite_strings["OR"])] + + return q_string + + def basic_query(self, query): + + """Basic text query on SQLite using FTS5 index""" + + query_str = self._prep_query(query) + + sql_query = f"SELECT rank, rowid, * FROM {self.library_name} " \ + f"WHERE text_search MATCH '{query_str}' ORDER BY rank" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack_search_result(results) + + self.conn.close() + + return output + + def filter_by_key(self, key, value): + + """Returns rows in which col (key) = value""" + + # used for getting library card + sql_query = f"SELECT rowid, * FROM {self.library_name} WHERE {key} = '{value}';" + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def text_search_with_key_low_high_range(self, query, key, low, high, key_value_dict=None): + + """Text search with additional filter of col (key) in low to high value range specified""" + + query_str = self._prep_query(query) + + sql_query = f"SELECT rank, rowid, * FROM {self.library_name} WHERE text_search MATCH '{query_str}' " \ + f"AND {key} BETWEEN {low} AND {high}" + + if key_value_dict: + for key, value in key_value_dict.items(): + sql_query += f" AND {key} = {value}" + + sql_query += " ORDER BY rank" + sql_query += ";" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack_search_result(results) + + self.conn.close() + + return output + + def text_search_with_key_value_range(self, query, key, value_range_list, key_value_dict=None): + + """Text search with additional filter of key in value_range list with optional further key=value pairs + in key_value_dict""" + + query_str = self._prep_query(query) + + # insert_array = (tuple(value_range_list),) + # insert_array = tuple(value_range_list) + # ia_str = str("(1)") + # insert_array = value_range_list + # insert_array = [1,] + + ia_str = "(" + for v in value_range_list: + if isinstance(v, int): + ia_str += str(v) + else: + ia_str += "'" + v + "'" + ia_str += ", " + if ia_str.endswith(", "): + ia_str = ia_str[:-2] + ia_str += ")" + + sql_query = f"SELECT rank, rowid, * FROM {self.library_name} WHERE text_search MATCH '{query_str}' " \ + f"AND {key} IN {ia_str}" + + if key_value_dict: + for key, value in key_value_dict.items(): + sql_query += f" AND {key} = '{value}'" + + sql_query += " ORDER BY rank" + + sql_query += ";" + + results = self.conn.cursor().execute(sql_query) + output = self.unpack_search_result(results) + + self.conn.close() + + return output + + def text_search_with_key_value_dict_filter(self, query, key_value_dict): + + """Text search with additional 'AND' filter of key=value for all keys in key_value_dict""" + + query_str = self._prep_query(query) + + sql_query = f"SELECT rank, rowid, * FROM {self.library_name} WHERE text_search MATCH '{query_str}' " + + insert_array = () + + if key_value_dict: + for key, value in key_value_dict.items(): + + if isinstance(value,list): + + sql_query += f" AND (" + + for items in value: + if isinstance(value,str): + sql_query += f" {key} = '{items}' OR " + else: + sql_query += f" {key} = {items} OR " + + if sql_query.endswith("OR "): + sql_query = sql_query[:-3] + sql_query += ")" + + else: + if isinstance(value,str): + sql_query += f" AND {key} = '{value}'" + else: + sql_query += f" AND {key} = {value}" + + sql_query += " ORDER BY rank" + sql_query += ";" + + results = self.conn.cursor().execute(sql_query, insert_array) + output = self.unpack_search_result(results) + + self.conn.close() + + return output + + def get_distinct_list(self, key): + + """Gets distinct elements from list for selected col (key)""" + + sql_query = f"SELECT DISTINCT {key} FROM {self.library_name};" + results = self.conn.cursor().execute(sql_query) + + output = [] + for res in results: + if res: + if len(res) > 0: + output.append(res[0]) + + self.conn.close() + + return output + + def filter_by_key_dict (self, key_dict): + + """Filters and returns elements where key=value as specified by the key_dict""" + + sql_query = f"SELECT rowid, * FROM {self.library_name}" + + conditions_clause = " WHERE" + for key, value in key_dict.items(): + + # handles passing a filter with 'mongo' style $in key range + if isinstance(value,dict): + if "$in" in value: + value = value["$in"] + + logger.debug(f"update: SQLite - filter_by_key_dict - value - {value}") + + if isinstance(value,list): + v_str = "(" + for entry in value: + v_str += str(entry) + "," + if v_str.endswith(","): + v_str = v_str[:-1] + v_str += ")" + conditions_clause += f" {key} IN {v_str} AND " + else: + if isinstance(value, int): + conditions_clause += f" {key} = {value} AND " + else: + conditions_clause += f" {key} = '{value}' AND " + + # conditions_clause += f" {key} = {value} AND " + + if conditions_clause.endswith(" AND "): + conditions_clause = conditions_clause[:-5] + + if len(conditions_clause) > len(" WHERE"): + sql_query += conditions_clause + ";" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def filter_by_key_value_range(self, key, value_range): + + """Filter by key in value range, e.g., {"doc_ID": [1,2,3,4,5]}""" + + value_range_str = "(" + + for v in value_range: + value_range_str += "'" + str(v) + "'" + ", " + + if value_range_str.endswith(", "): + value_range_str = value_range_str[:-2] + + value_range_str += ")" + + sql_query = f"SELECT rowid, * from {self.library_name} WHERE {key} IN {value_range_str};" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def filter_by_key_ne_value(self, key, value): + + """Filters where key not equal to value""" + + sql_query = f"SELECT rowid, * from {self.library_name} WHERE NOT {key} = {value};" + + results = self.conn.cursor().execute(sql_query) + + output = self.unpack(results) + + self.conn.close() + + return output + + def embedding_job_cursor(self, new_embedding_key, doc_id=None): + + """Handles end-to-end retrieval of text blocks to be embedded - returns Cursor""" + + if doc_id: + # pull selected documents for embedding + insert_array = () + insert_array += (tuple(doc_id),) + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE doc_ID IN %s;" + results = list(self.conn.cursor().execute(sql_query, insert_array)) + count = results[0] + + sql_query = f"SELECT rowid, * FROM {self.library_name} WHERE doc_ID IN %s;" + results = self.conn.cursor().execute(sql_query, insert_array) + + else: + + # Note: for SQLite - only designed for single embedding, not multiple embeddings on each block + + # first get the total count of blocks 'un-embedded' with this key in the collection + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE embedding_flags IS NULL OR " \ + f"embedding_flags != '{new_embedding_key}';" + + results = list(self.conn.cursor().execute(sql_query)) + count = results[0] + + sql_query = f"SELECT rowid, * FROM {self.library_name} WHERE embedding_flags IS NULL OR " \ + f"embedding_flags != '{new_embedding_key}';" + + results = self.conn.cursor().execute(sql_query) + + results = list(results) + + cursor = DBCursor(results, self, "sqlite") + + return count[0], cursor + + def count_embedded_blocks(self, embedding_key): + + """Count all blocks to be embedded in current job scope""" + + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE embedding_flags = '{embedding_key}';" + + results = list(self.conn.cursor().execute(sql_query)) + + embedded_blocks = results[0] + + self.conn.close() + + return embedded_blocks[0] + + def count_documents(self, filter_dict): + + """Count documents that match filter conditions""" + + conditions_clause = "" + + if filter_dict: + + for key, value in filter_dict.items(): + conditions_clause += f"{key} = '{value}' AND " + + if conditions_clause.endswith(" AND "): + conditions_clause = conditions_clause[:-5] + + if conditions_clause: + sql_query = f"SELECT COUNT(*) FROM {self.library_name} WHERE {conditions_clause};" + else: + sql_query = f"SELECT COUNT(*) FROM {self.library_name};" + + results = list(self.conn.cursor().execute(sql_query)) + + output = results[0] + + self.conn.close() + + return output + + def close(self): + """Closes SQLite connection""" + self.conn.close() + return 0 + + def direct_custom_query(self, query_filter): + """Applies the custom query directly to the DB and returns the results""" + results = list(self.conn.cursor().execute(str(query_filter))) + # output = self.unpack(results) + self.conn.close() + return results + + def list_all_tables(self): + + """Get list of all tables on the database""" + + sql_query = f"SELECT * FROM sqlite_master WHERE type = 'table';" + + results = self.conn.cursor().execute(sql_query) + + list_of_tables = list(self.conn.cursor().execute(sql_query)) + + self.conn.close() + + return list_of_tables + + def get_schema(self, table_name): + + """ Lookup of table_schema for an input table_name - outputs 'create table schema string' that can + be used directly as context in a text2sql inference """ + + table_schema = "" + schema_dict = {} + primary_key = "" + + sql_query = f"SELECT * FROM sqlite_master WHERE type = 'table' AND name = '{table_name}';" + + table_schema_row = self.conn.cursor().execute(sql_query) + table_schema_row = list(table_schema_row) + + if len(table_schema_row) > 0: + + # in [0][4] is the original sql command to create the table, + # e.g., 'CREATE TABLE {table_name} (key1 datatype1, key2 datatype2, key3 datatype3, ...)' + table_schema = table_schema_row[0][4] + + # split to look only at the string portion inside ( ) + ts_split = table_schema.split("(")[-1] + ts_split = ts_split.split(")")[0] + + # split by ',' to get the individual chunks with key datatype + ts_chunks = ts_split.split(",") + + for chunk in ts_chunks: + + found_primary_key = False + + if chunk.strip(): + + if chunk.strip().endswith("PRIMARY KEY"): + found_primary_key = True + chunk = chunk.strip()[:-len("PRIMARY KEY")] + + # remove any leading/trailing spaces and split by space + kv = chunk.strip().split(" ") + + # if well formed should be only two values - excluding PRIMARY KEY ! + if len(kv) == 2: + schema_dict.update({kv[0]:kv[1]}) + + if found_primary_key: + primary_key = kv[0] + + if primary_key: + schema_dict.update({"PRIMARY KEY": primary_key}) + + return schema_dict + + +class SQLiteWriter: + + """SQLiteWriter is the main class abstraction to handle writes, indexes, edits and deletes on SQLite DB""" + + def __init__(self, library_name, account_name="llmware", custom_table=False, custom_schema=None): + + self.library_name = library_name + self.account_name = account_name + + self.conn = _SQLiteConnect().connect(library_name) + + if library_name == "status": + self.schema = LLMWareTableSchema().get_status_schema() + + elif library_name == "library": + self.schema = LLMWareTableSchema().get_library_card_schema() + + elif library_name == "parser_events": + self.schema = LLMWareTableSchema().get_parser_table_schema() + + else: + self.schema = LLMWareTableSchema().get_block_schema() + + self.reserved_tables = ["status", "library", "parser_events"] + self.text_retrieval = False + + if library_name not in self.reserved_tables: + self.text_retrieval = True + + self.custom_table = custom_table + + if custom_table: + self.text_retrieval = False + if custom_schema: + self.schema = custom_schema + + def build_text_index(self, index_col="text_search"): + + """No separate text index created on SQLite - created in Virtual Table at time of set up""" + + return True + + def check_if_table_build_required(self): + + """Checks if table exists, and if not, responds True that build is required""" + + sql_query = f"SELECT * FROM sqlite_master " \ + f"WHERE type = 'table' AND name = '{self.library_name}';" + + results = self.conn.cursor().execute(sql_query) + + if len(list(results)) > 0: + table_build = False + else: + table_build = True + + return table_build + + def _build_sql_virtual_table_from_schema (self, table_name, schema): + + table_create = f"CREATE VIRTUAL TABLE IF NOT EXISTS {table_name} USING fts5(" + + for key, value in schema.items(): + + if key not in ["_id", "PRIMARY KEY"]: + table_create += key + ", " + + if table_create.endswith(", "): + table_create = table_create[:-2] + + table_create += ");" + + return table_create + + def _build_sql_from_schema (self, table_name, schema): + + """Builds SQL table create string from schema dictionary""" + + table_create = f"CREATE TABLE IF NOT EXISTS {table_name} (" + + for key, value in schema.items(): + + # replace jsonb with json for sqlite + if value == "jsonb": + value = "json" + + table_create += key + " " + value + ", " + + if table_create.endswith(", "): + table_create = table_create[:-2] + + table_create += ");" + + return table_create + + def create_table(self, table_name, schema, add_search_column=True): + + """Builds SQL table""" + + if table_name not in ["library", "status", "parsing_events"] and not self.custom_table: + + # used for creating library text search index + table_create = self._build_sql_virtual_table_from_schema(table_name, schema) + + else: + # status, library, parser_events + any other structured table + table_create = self._build_sql_from_schema(table_name, schema) + + self.conn.execute(table_create) + self.conn.commit() + + # close connection at end of update + self.conn.close() + + return 1 + + def write_new_record(self, new_record): + + """Writes new record - primary for creating new library card and status update""" + + keys_list = "(" + output_values = "(" + + for keys, values in new_record.items(): + keys_list += keys + ", " + + new_entry = str(new_record[keys]) + new_entry = new_entry.replace("'", '"') + + output_values += "'" + new_entry + "'" + ", " + + if keys_list.endswith(", "): + keys_list = keys_list[:-2] + + if output_values.endswith(", "): + output_values = output_values[:-2] + + keys_list += ")" + output_values += ")" + + sql_instruction = f"INSERT INTO {self.library_name} {keys_list} VALUES {output_values};" + + results = self.conn.cursor().execute(sql_instruction) + + self.conn.commit() + + self.conn.close() + + return 1 + + def write_new_parsing_record(self, rec): + + """ Writes new parsing record dictionary into SQLite """ + + sql_string = f"INSERT INTO {self.library_name}" + sql_string += " (block_ID, doc_ID, content_type, file_type, master_index, master_index2, " \ + "coords_x, coords_y, coords_cx, coords_cy, author_or_speaker, added_to_collection, " \ + "file_source, table_block, modified_date, created_date, creator_tool, external_files, " \ + "text_block, header_text, text_search, user_tags, special_field1, special_field2, " \ + "special_field3, graph_status, dialog, embedding_flags) " + sql_string += " VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, " \ + "$19, $20, $21, $22, $23, $24, $25, $26, $27, $28);" + + # now unpack the new_record into parameters + insert_arr = (rec["block_ID"], rec["doc_ID"],rec["content_type"], rec["file_type"], rec["master_index"], + rec["master_index2"], rec["coords_x"], rec["coords_y"], rec["coords_cx"], rec["coords_cy"], + rec["author_or_speaker"], rec["added_to_collection"], rec["file_source"], rec["table"], + rec["modified_date"], rec["created_date"], rec["creator_tool"], rec["external_files"], + rec["text"], rec["header_text"], rec["text_search"], rec["user_tags"], + rec["special_field1"], rec["special_field2"], rec["special_field3"], rec["graph_status"], + rec["dialog"], "") + + # note: sets embedding flag - parameter $28 to "" by default + + results = self.conn.cursor().execute(sql_string,insert_arr) + + self.conn.commit() + + self.conn.close() + + return True + + def destroy_collection(self, confirm_destroy=False): + + """Drops table""" + + if confirm_destroy: + + sql_instruction = f"DROP TABLE {self.library_name};" + + # returns TRUE if table does not exist & FALSE if table exists + table_does_not_exist = self.check_if_table_build_required() + + # if FALSE ... drop the table + if not table_does_not_exist: + results = self.conn.cursor().execute(sql_instruction) + self.conn.commit() + self.conn.close() + return 1 + else: + logger.warning(f"update: SQLiteWriter - request to drop table not executed because table " + f"could not be found in the database.") + return -1 + + logger.warning("update: library not destroyed - need to set confirm_destroy = True") + self.conn.close() + + return 0 + + def update_block (self, doc_id, block_id, key, new_value, default_keys): + + """Updates block by specified (doc_id, block_id) pair""" + + completed = False + + if key in default_keys: + + sql_instruction = f"UPDATE {self.library_name} "\ + f"SET {key} = {new_value} " \ + f"WHERE doc_ID = {doc_id} AND block_ID = {block_id};" + + completed = True + results = self.conn.cursor().execute(sql_instruction) + + self.conn.commit() #added: otherwise it wouldn't write to the db! + + self.conn.close() + + return completed + + def update_one_record(self, filter_dict, key, new_value): + + """Updates one record""" + + conditions_clause = "" + for k, v in filter_dict.items(): + conditions_clause += f" {k} = '{v}' AND" # added a space between start of quote and {k} otherwise broken + + if conditions_clause.endswith(" AND"): + conditions_clause = conditions_clause[:-4] + + if conditions_clause: + sql_instruction = f"UPDATE {self.library_name} " \ + f"SET {key} = '{new_value}' " \ + f"WHERE {conditions_clause};" + + results = self.conn.cursor().execute(sql_instruction) + self.conn.commit() + + self.conn.close() + + return 0 + + def replace_record(self, filter_dict, new_entry): + + """Check if existing record with the same key - if so, delete, then create new""" + + new_values = "(" + for keys, values in new_entry.items(): + + if keys not in ["_id"]: + new_values += "'" + str(values) + "', " + + if new_values.endswith(", "): + new_values = new_values[:-2] + new_values += ")" + + conditions_clause = "" + for keys, values in filter_dict.items(): + conditions_clause += f"{keys} = '{values}' AND" + if conditions_clause.endswith(" AND"): + conditions_clause = conditions_clause[:-4] + + sql_check = f"SELECT * FROM {self.library_name} WHERE {conditions_clause};" + + exists = list(self.conn.cursor().execute(sql_check)) + + if exists: + # need to delete, then replace with new record + sql_delete = f"DELETE FROM {self.library_name} WHERE {conditions_clause};" + self.conn.cursor().execute(sql_delete) + + sql_new_insert = f"INSERT INTO {self.library_name} VALUES {new_values};" + + self.conn.cursor().execute(sql_new_insert) + self.conn.commit() + + self.conn.close() + + return 0 + + def delete_record_by_key(self, key, value): + + """Deletes record by matching key = value""" + + sql_command = f"DELETE FROM {self.library_name} WHERE {key} = '{value}';" + self.conn.execute(sql_command) + self.conn.commit() + self.conn.close() + return 0 + + def update_library_card(self, library_name, update_dict, lib_card, delete_record=False): + + """Updates library card""" + + conditions_clause = f"library_name = '{library_name}'" + update_clause = "" + + for key, new_value in update_dict.items(): + + if key != "embedding": + + if isinstance(new_value, int): + update_clause += f"{key} = {new_value}, " + else: + update_clause += f"{key} = '{new_value}', " + else: + # will update in second step + current_emb_record = lib_card["embedding"] + embedding_update = self._update_embedding_record_handler(current_emb_record, new_value, + delete_record=delete_record) + # from pg- start + embedding_update=str(embedding_update) + embedding_update=embedding_update.replace("'", '"') + update_clause += f"{key} = '{embedding_update}', " + # from pg- end + + if update_clause.endswith(", "): + update_clause = update_clause[:-2] + + sql_instruction = f"UPDATE {self.library_name} " \ + f"SET {update_clause} " \ + f"WHERE {conditions_clause};" + + self.conn.cursor().execute(sql_instruction) + self.conn.commit() + + self.conn.close() + + return 1 + + def _update_embedding_record_handler(self, embedding_list, new_value, delete_record=False): + + """Internal helper method to integrate embedding update into array of dicts- which + is inserted as JSON directly in Postgres""" + + # special flag to identify where to 'merge' and update an existing embedding record + merged_embedding_update = False + inserted_list = [] + + if len(embedding_list) > 0: + # if the last row is a "no" embedding, then remove it + if embedding_list[-1]["embedding_status"] == "no": + del embedding_list[-1] + + for emb_records in embedding_list: + + if emb_records["embedding_model"] == new_value["embedding_model"] and \ + emb_records["embedding_db"] == new_value["embedding_db"]: + + if not delete_record: + inserted_list.append(new_value) + else: + pass + + merged_embedding_update = True + + # catch potential error + + if not delete_record: + if "embedded_blocks" in emb_records and "embedded_blocks" in new_value: + + if emb_records["embedded_blocks"] > new_value["embedded_blocks"]: + logger.warning(f"warning: may be issue with embedding - mis-alignment in " + f"embedding block count - {emb_records['embedded_blocks']} > " + f"{new_value['embedded_blocks']}") + + else: + inserted_list.append(emb_records) + + if not merged_embedding_update: + embedding_list.append(new_value) + + output = embedding_list + else: + output = inserted_list + + return output + + def get_and_increment_doc_id(self, library_name): + + """Gets and increments unique doc ID""" + + val_out = -1 + + val_array = (str(library_name),) + + sql_instruction = f"UPDATE library " \ + f"SET unique_doc_id = unique_doc_id + 1 " \ + f"WHERE library_name = '{library_name}' " \ + f"RETURNING unique_doc_id" + + result = self.conn.cursor().execute(sql_instruction) + + output = list(result) + if len(output) > 0: + val = output[0] + if len(val) > 0: + val_out = val[0] + + self.conn.commit() + self.conn.close() + + return val_out + + def set_incremental_docs_blocks_images(self, library_name, added_docs=0, added_blocks=0, added_images=0, + added_pages=0, added_tables=0): + + """Increments key counters on library card post parsing""" + + conditions_clause = f"library_name = '{library_name}'" + + set_clause = f"documents = documents + {added_docs}, " \ + f"blocks = blocks + {added_blocks}, " \ + f"images = images + {added_images}, " \ + f"pages = pages + {added_pages}, " \ + f"tables = tables + {added_tables}" + + sql_instruction = f"UPDATE {self.library_name} SET {set_clause} WHERE {conditions_clause};" + + results = self.conn.cursor().execute(sql_instruction) + + self.conn.commit() + + self.conn.close() + + return 0 + + def add_new_embedding_flag(self, _id, embedding_key, value): + + """SQLite implementation saves new embedding flag in column and replaces any previous values""" + + # the embedding key name is saved in embedding_flags, and the index is saved in special_field1 + + insert_array = () + + insert_array += (embedding_key,) + value=str(value) + + sql_command = f"UPDATE {self.library_name} " \ + f"SET embedding_flags = '{embedding_key}', special_field1 = '{value}' " \ + f"WHERE rowid = {_id}" + + self.conn.cursor().execute(sql_command) + self.conn.commit() + + self.conn.close() + + return 0 + + def unset_embedding_flag(self, embedding_key): + + """To complete deletion of an embedding, remove the json embedding_key from the text collection""" + + sql_instruction = f"UPDATE {self.library_name} " \ + f"SET embedding_flags = ''" \ + f"WHERE embedding_flags = '{embedding_key}';" + + self.conn.cursor().execute(sql_instruction) + self.conn.commit() + self.conn.close() + + return 0 + + def close(self): + """Closes SQLite connection""" + self.conn.close() + return 0 + + +class _PGConnect: + + """_PGConnect returns a Postgres DB connection""" + + def __init__(self): + + # Connect to postgres + self.postgres_host = None + self.postgres_port = None + self.postgres_db_name = None + self.postgres_user_name = None + self.postgres_full_schema = None + self.postgres_pw = None + + self.conn = None + + def connect(self, db_name=None, collection_name=None): + + """Connect to Postgres DB - using config data in PostgresConfig""" + + self.postgres_host = PostgresConfig().get_config("host") + self.postgres_port = PostgresConfig().get_config("port") + self.postgres_user_name = PostgresConfig().get_config("user_name") + self.postgres_pw = PostgresConfig().get_config("pw") + + # postgres will use the configured db name + # if db_name: self.postgres_db_name = db_name + + self.postgres_db_name = PostgresConfig().get_config("db_name") + + self.conn = psycopg.connect(host=self.postgres_host, port=self.postgres_port, dbname=self.postgres_db_name, + user=self.postgres_user_name, password=self.postgres_pw) + + return self.conn + + +class _SQLiteConnect: + + """_SQLiteConnect returns a connection to a SQLite DB running locally""" + + def __init__(self): + self.conn = None + + # check for llmware path & create if not already set up, e.g., "first time use" + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + logger.info("_SQliteConnect - Setting up LLMWare Workspace.") + + def connect(self, db_name=None, collection_name=None): + + """Connect to SQLite DB - using configuration parameters in SQLiteConfig""" + + # db_file = os.path.join(SQLiteConfig.get_db_fp(), "sqlite_llmware.db") + db_file = SQLiteConfig.get_uri_string() + + self.conn = sqlite3.connect(db_file) + + return self.conn + + +class _MongoConnect: + + """_MongoConnect returns a connection to a Mongo collection""" + + def __init__(self): + + # self.collection_db_path = LLMWareConfig.get_config("collection_db_uri") + self.collection_db_path = LLMWareConfig.get_db_uri_string() + # self.client = MongoClient(self.collection_db_path, unicode_decode_error_handler='ignore') + self.mongo_client = None + self.timeout_secs = 5 + + def connect(self, db_name=None,collection_name=None): + + """Connect to Mongo DB collection""" + + self.mongo_client = MongoDBManager().client[db_name][collection_name] + return self.mongo_client + + +class MongoDBManager: + + """This is internal class - recommended as best practice by Mongo to manage connection threads""" + + class __MongoDBManager: + + def __init__(self): + + # self.collection_db_path = LLMWareConfig.get_config("collection_db_uri") + self.collection_db_path = LLMWareConfig.get_db_uri_string() + + # default client is Mongo currently + self.client = MongoClient(self.collection_db_path, unicode_decode_error_handler='ignore') + # self.client.admin.authenticate(username, password) + + __instance = None + + def __init__(self): + + if not MongoDBManager.__instance: + MongoDBManager.__instance = MongoDBManager.__MongoDBManager() + + def __getattr__(self, item): + return getattr(self.__instance, item) + + +class DBCursor: + + """Wrapper class around database cursors to handle specific cursor management across DBs""" + + def __init__(self, cursor, collection_retriever, db_name, close_when_exhausted=True, return_dict=True, schema=None): + + self.cursor = iter(cursor) + self.collection_retriever = collection_retriever + self.db_name = db_name + self.close_when_exhausted = close_when_exhausted + self.return_dict = return_dict + self.schema = schema + + def pull_one(self): + + """Calls next on the iterable cursor to pull one new row off the cursor and return to calling function""" + + try: + new_row = next(self.cursor) + + except StopIteration: + # The cursor is empty (no blocks found) + new_row = None + if self.close_when_exhausted: + self.collection_retriever.close() + + if new_row and self.return_dict and not isinstance(new_row,dict): + return self.collection_retriever.unpack([new_row])[0] + + return new_row + + def pull_all(self): + + """Exhausts remaining cursor and returns to calling function""" + + output = [] + + while True: + + new_entry = self.pull_one() + + if not new_entry: + break + else: + output.append(new_entry) + + """ + for entries in self.cursor: + + if self.return_dict and not isinstance(entries,dict): + output.append(self.collection_retriever.unpack(entries)) + else: + output.append(entries) + """ + + return output + + +class CustomTable: + + """ CustomTable resource that can be implemented on a LLMWare collection database using consistent set of methods + and utilities. Intended for creation of supporting tables and leveraging structured and semi-structured + information in conjunction with LLM-based workflows and for easier integration of supporting tables in + larger projects. """ + + def __init__(self, db=None, table_name=None, schema=None, library_name=None, account_name="llmware", + auto_correct_schema_errors=True,auto_correct_postpend="_d"): + + # check for llmware path & create if not already set up, e.g., "first time use" + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + logger.info("CustomTable - Setting up LLMWare Workspace.") + + if not db: + self.db = LLMWareConfig().get_active_db() + else: + if db in LLMWareConfig().get_supported_collection_db(): + self.db = db + + self.reserved_tables = ["status", "library", "parser_events"] + + # this list will be improved over time to include common reserved col names + self.reserved_col_names = ["table","text"] + + if table_name not in self.reserved_tables: + self.table_name = table_name + else: + logger.warning( + f"error: proposed custom table name - {table_name} - is a reserved table name and can not be used. " + f"self.table_name is being set to None, and will need to be set to a different name before using.") + + self.table_name = None + + self.schema = schema + self.library_name = library_name + self.account_name = account_name + self.db_connection = None + + # if schema column name is in reserved list, then will add the value of schema_postpend to end of string + self.auto_correct_schema_errors = auto_correct_schema_errors + self.postpend = auto_correct_postpend + + # attributes used when loading a custom csv or json/jsonl to populate a table + self.rows = None + self.col_numbers = [] + self.col_names = [] + self.column_map = {} + self.sql_create_table = None + + # check if table_name already registered in LLMWareTableSchema + if table_name in LLMWareTableSchema().get_custom_tables() and not self.schema: + + self.schema = LLMWareTableSchema().get_custom_schema()[table_name] + + # check if table already created in DB, and if not, create table + if table_name and self.schema: + + if self.db != "mongo": + self.build_table() + + # confirm that table name and schema are registered in LLMWareTableSchema for easy future access + if self.table_name and self.schema: + LLMWareTableSchema().register_custom_schema(self.table_name, self.schema, replace=False) + + def build_table(self, table_name=None, schema=None): + + """ Method to explicitly build a new table with the table_name and schema passed. This is not needed if + both table_name and schema are passed in the constructor when instantiating the CustomTable. """ + + if table_name: + self.table_name = table_name + + if schema: + self.schema = schema + + completed = False + + if self.table_name and self.schema: + + # runs a quick check to identify common schema errors + # if auto_correct == True, then will attempt to remediate + confirmation = self._check_and_remediate_schema() + + if confirmation: + + conn = self.get_connection(table_name=self.table_name, type="write") + if conn.check_if_table_build_required(): + conn.create_table(self.table_name, self.schema) + conn.close() + + completed = True + + else: + logger.warning(f"warning: could not successfully remediate schema - there is an issue with the " + f"current schema - please review and adjust.") + complete = False + + else: + logger.warning(f"warning: could not build_table since missing either table name, schema or both - " + f"table_name = {self.table_name} - schema = {self.schema}") + completed = False + + return completed + + def _check_and_remediate_schema(self, schema=None): + + """ Internal method used in table build process to check that namespace of proposed schema do not conflict + with any reserved terms or have other potential issues. If auto_correct_schema_errors == True, then + it will attempt to fix the schema and self.rows on the fly by identifying reserved keywords, and + adding the designated 'postpend' value to the term, e.g., "_d" + + If auto_correct_schema_errors == False, and there are schema errors, then confirmation returned will be False. + + """ + + if schema: + self.schema = schema + + confirmation = True + + updated_schema = {} + remediated_keys = [] + + for k, v in self.schema.items(): + if k in self.reserved_col_names: + logger.warning(f"warning: schema column name - {k} - in reserved column names list - will need to " + f"change before creating table.") + updated_schema.update({k+self.postpend: v}) + remediated_keys.append(k) + confirmation = False + else: + updated_schema.update({k:v}) + + if not confirmation: + if self.auto_correct_schema_errors: + + # remediated problematic names + self.schema = updated_schema + + # change keys in self.rows + updated_rows = [] + for x in range(0,len(self.rows)): + new_row = {} + for k,v in self.rows[x].items(): + if k in self.reserved_col_names: + new_row.update({k+self.postpend: v}) + else: + new_row.update({k:v}) + updated_rows.append(new_row) + + self.rows = updated_rows + confirmation = True + + return confirmation + + def get_connection(self, table_name=None, type="read"): + + """ Convenience method that gets a connection to the selected database with two connection types - + 'read' and 'write'. """ + + if table_name: + self.table_name = table_name + + if type == "read": + self.db_connection = CollectionRetrieval(self.table_name, account_name=self.account_name, + db_name=self.db, custom_table=True, custom_schema=self.schema) + + if type == "write": + self.db_connection = CollectionWriter(self.table_name, account_name=self.account_name, + db_name=self.db, custom_table=True, custom_schema=self.schema) + + if type not in ["read", "write"]: + raise LLMWareException(message=f"Exception: not recognized connection type {type}") + + return self.db_connection + + def close_connection(self): + + """ Used by each method that opens a connection to explicitly close the connection at the end of the + processing. """ + + self.db_connection.close() + return True + + def _get_best_guess_value_type(self, v): + + """ Simple utility function to check the value of a 'test row' element and use as the basis for an automated + data type determination. This can be replaced/supplemented with more sophisticated checks, or the data + type can be explicitly passed as a more robust alternative. """ + + # all data elements will be set to either: "text" | "float" | "integer" + + dt = "text" + + try: + # if whole number, then store in DB as 'integer' + vt = int(v) + dt = "integer" + if self.db == "postgres": + # for safety, pick the largest int data type + dt = "bigint" + + # if there is a 0 value, then use float to be safe + if vt == 0: + dt = "float" + + except: + try: + # if element can be converted to a python 'float', then store in DB as 'float' + # may evaluate if 'decimal' or 'numeric' is better default choice + vt = float(v) + dt = "float" + except: + # if can not convert to a python number, then store in DB as 'text' + dt = "text" + + return dt + + def test_and_remediate_schema(self, samples=10, auto_remediate=True): + + """ Applies a larger test of the schema against the data held in CustomTable .rows attribute - will + test the number of samples passed as an optional parameter. + + If auto_remediate == True (default), then it will automatically update the data type used in the schema to + the 'safest' among the types found in the sample set. """ + + updated_schema = {} + + for key, data_type in self.schema.items(): + + if len(self.rows) < samples: + samples = len(self.rows) + + samples_dt_list = [] + + for x in range(0,samples): + + if key not in self.rows[x]: + logger.warning(f"warning: CustomTable - test_and_remediate_schema - unexpected error - " + f"key not found in row - {x} - {key} - {self.rows[x]}") + else: + check_value = self.rows[x][key] + samples_dt_list.append(self._get_best_guess_value_type(check_value)) + + # simple decision tree - will add more options over time + if "text" in samples_dt_list or data_type=="text": + dt = "text" + elif "float" in samples_dt_list or data_type=="float": + dt = "float" + elif "integer" in samples_dt_list or "bigint" in samples_dt_list and data_type in ["integer", "bigint"]: + dt = "integer" + if self.db == "postgres": + dt = "bigint" + else: + # when in doubt, assign 'text' as safest choice + dt = "text" + + updated_schema.update({key: dt}) + + if auto_remediate: + self.schema = updated_schema + + return updated_schema + + def load_json(self, fp, fn, selected_keys=None, data_type_map=None, schema=None): + + """ Import JSON/JSONL file - build schema and 'rows' to be processed and loaded to DB. Assumes that + JSON/JSONL file is a well-formed 'pseudo-DB' with each entry consisting of elements with the same + dictionary keys. + + -- If an optional list of selected_keys is passed, then it will be used to filter the elements + found in the table, and only 'keys' in the 'selected_keys' list will be added to the schema. + + --If an optional (dictionary) key_data_type_map is passed, e.g., {"account_ID": "decimal"}, then the + data type passed will be used in the database schema. If no explicit data type provided, then + it will use 'best guess' to determine a safe type. + + -- Assumes that entire file can be read in memory. This is intended for + small tables and rapid prototyping - and not as a general purpose DB loading utility. """ + + f = open(os.path.join(fp, fn), "r", encoding='utf-8-sig', errors='ignore') + + if fn.endswith(".json"): + json_rows = json.load(f) + + elif fn.endswith(".jsonl"): + json_rows = [] + for entries in f: + row = json.loads(entries) + json_rows.append(row) + else: + raise LLMWareException(message=f"Exception: File Type - {fn} - not supported - please use a .json or " + f".jsonl file with this method.") + + # get to list of rows, with each comprising a dictionary of keys + if len(json_rows) == 0: + raise LLMWareException(message="Exception: length of json rows == 0 - no content found suitable " + "for DB ingestion.") + + # assume that json_rows will be a list of dictionaries as default case + if isinstance(json_rows, list): + first_row = json_rows[0] + else: + # exception case - if inserting only 'one' row into DB with a small JSON file + first_row = json_rows + + self.rows = [] + skipped_rows = [] + + if not schema: + + # if no schema provided, then derive from data structure implicitly + + schema = {} + + for keys, values in first_row.items(): + + add_key = True + if selected_keys: + if keys not in selected_keys: + add_key = False + + if add_key: + + dt = "text" + if data_type_map: + if keys in data_type_map: + dt = data_type_map[keys] + else: + dt = self._get_best_guess_value_type(values) + else: + dt = self._get_best_guess_value_type(values) + + schema.update({keys:dt}) + + # assigns both the updated schema and rows to the attributes of the CustomTable + self.schema = schema + + column_size = len(self.schema.items()) + + # iterate thru all rows (and extract keys matching to schema from each row) + for x in range(0,len(json_rows)): + + new_row = {} + + for k, v in json_rows[x].items(): + + if k in self.schema: + new_row.update({k:v}) + + if len(new_row.items()) != column_size: + logger.warning(f"warning: on line - {x} - extracted elements - {len(new_row.items())} - does " + f"not map to the target column size - {column_size} - skipping row - {new_row}.") + skipped_rows.append([x, json_rows[x]]) + + else: + self.rows.append(new_row) + + output = {"rows": len(self.rows), "columns": column_size, "schema": self.schema, + "skipped_rows": skipped_rows} + + return output + + @staticmethod + def file_load (in_path, delimiter=",",encoding='utf-8-sig',errors='ignore'): + + """ Utility function provided inline - mirrors function in Utility class - basic configurable + csv file loading with optional parameters to set delimiter (e.g., ',' '\t', etc.) and encoding. + + Returns a python list of lists in which each element corresponds to a 'row' of the spreadsheet, and is in turn, + a list of each of the individual 'cells' of the csv. """ + + record_file = open(in_path, encoding=encoding,errors=errors) + c = csv.reader(record_file, dialect='excel', doublequote=False, delimiter=delimiter) + output = [] + for lines in c: + output.append(lines) + record_file.close() + + return output + + def validate_json(self, fp, fn, key_list=None): + + """ Provides an assessment and validation of a json/jsonl file and readiness for insertion into a + database. Optional key_list can be passed to confirm validation of a specified (minimum) set of + keys present in every row of the file. + + If no key_list provided, then the keys in the first row of the + file will be used as the basis for all future rows. + + Output is a dictionary with analysis of rows, columns, keys, and any nonconforming rows. + + *** This method is intended to be used prior to loading a JSON/JSONL file into a CustomTable to assess + readiness and any additional preparation steps. *** """ + + f = open(os.path.join(fp, fn), "r", encoding='utf-8-sig', errors='ignore') + + if fn.endswith(".json"): + json_rows = json.load(f) + + elif fn.endswith(".jsonl"): + json_rows = [] + for entries in f: + row = json.loads(entries) + json_rows.append(row) + else: + raise LLMWareException(message=f"Exception: File Type - {fn} - not supported - please use a .json or " + f".jsonl file with this method.") + + col_count_tracker = {} + noted_list = [] + total_rows = len(json_rows) + all_keys_present_in_row = 0 + + if key_list: + keys = key_list + else: + keys= [] + + for x in range(0,total_rows): + + # if no key_list provided, then use the first row to identify the expected keys + if x == 0 and not key_list: + for k,v in json_rows[x].items(): + if k not in keys: + keys.append(k) + + if x >= 0: + all_keys_found = True + for key in keys: + if key not in json_rows[x]: + noted_list.append([x,key, json_rows[x]]) + all_keys_found = False + if all_keys_found: + all_keys_present_in_row += 1 + + row_elements = len(json_rows[x].items()) + if row_elements in col_count_tracker: + col_count_tracker[row_elements] += 1 + else: + col_count_tracker.update({row_elements: 1}) + + most_common_column = max(col_count_tracker, key=col_count_tracker.get) + match_percent = col_count_tracker[most_common_column] / total_rows + + if len(col_count_tracker.items()) > 1: + logger.warning(f"warning: found more than one length - {col_count_tracker}") + + for x in range(0,total_rows): + row_elements = len(json_rows[x].items()) + if row_elements != most_common_column: + noted_list.append([x,row_elements, json_rows[x]]) + + output = {"rows": total_rows, + "columns": most_common_column, + "rows_with_all_keys_present": all_keys_present_in_row, + "conforming_rows_percent": match_percent, + "column_analysis": col_count_tracker, + "keys": keys, + "nonconforming_rows": noted_list} + + return output + + def validate_csv(self, fp, fn, delimiter=',', encoding='utf-8-sig'): + + """ Provides an assessment and validation of a csv file and readiness for insertion into a + database. + + Output is a dictionary with analysis of rows, columns, and any nonconforming rows. + + *** This method is intended to be used prior to loading a CSV file into a CustomTable to assess + readiness and any additional preparation steps. *** """ + + # load csv + in_path = os.path.join(fp,fn) + output = self.file_load(in_path,delimiter=delimiter,encoding=encoding,errors='ignore') + + col_count_tracker = {} + noted_list = [] + total_rows = len(output) + + for x in range(0,total_rows): + row_elements = len(output[x]) + if row_elements in col_count_tracker: + col_count_tracker[row_elements] += 1 + else: + col_count_tracker.update({row_elements: 1}) + + most_common_column = max(col_count_tracker, key=col_count_tracker.get) + match_percent = col_count_tracker[most_common_column] / total_rows + + if len(col_count_tracker.items()) > 1: + logger.warning(f"warning: CustomTable - validate_csv - in reviewing the rows of the CSV - " + f"found more than one column length - {col_count_tracker}") + + for x in range(0,total_rows): + row_elements = len(output[x]) + if row_elements != most_common_column: + noted_list.append([x,row_elements, output[x]]) + + output = {"rows": total_rows, + "columns": most_common_column, + "conforming_rows_percent": match_percent, + "column_frequency_analysis": col_count_tracker, + "nonconforming_rows": noted_list} + + return output + + def load_csv(self, fp, fn, column_names=None, required_column_count=None, column_mapping_dict=None, + data_type_map=None, header_row=True, delimiter=',', encoding='utf-8-sig'): + + """ Import CSV file and extract schema and 'rows' to be loaded into DB. Assumes that CSV is a well-formed + 'pseudo-DB' with common set of elements in each row, and a first row that can be used to derive the + column names. + + -- if optional column_names is passed, then these names will be used instead of the first row of the + spreadsheet to derive names, with assumption that the column_names are in sequential order, and map 1:1 with + rows in the CSV. + + -- if an optional column_mapping_dict is passed, then this will be used to extract the key elements from the + CSV rows according to the mapping, e.g., + + column_mapping_dict = {"column1": 0, "column3": 2, "column5":4} + + This will extract 3 columns with the keys - "column1", "column3" and "column5" and will assign the values + in the 0th, 2nd, and 4th slots of each CSV row. + + -- if optional header_row is True, then the first row will be interpreted as a header row. If False, + then it will be interpreted as part of the dataset. """ + + # load csv + in_path = os.path.join(fp,fn) + output = self.file_load(in_path,delimiter=delimiter,encoding=encoding,errors='ignore') + + if (len(output) < 2 and header_row) or (len(output) < 1 and not header_row): + raise LLMWareException(message=f"Exception: not sufficient content found in the CSV to load " + f"into DB table - found {len(output)} rows") + + schema = {} + column_map = {} + self.rows = [] + skipped_rows = [] + + if column_mapping_dict: + column_map = column_mapping_dict + + else: + # get column names + if column_names: + + # will take this and map in sequential order + column_map = {} + for i, name in enumerate(column_names): + column_map.update({name:i}) + + else: + # will try to derive from the header row + if not header_row: + raise LLMWareException(message="Exception: can not determine the column names of the " + "spreadsheet - no 'column_names' passed, and there is no " + "header_row to use in the csv. To try to derive from the " + "CSV first row, set header_row = True.") + else: + + hrow = output[0] + + for i, entry in enumerate(hrow): + header_entry = re.sub("[\xfe\xff]", "", entry) + header_entry = re.sub(" ", "_", header_entry) + column_map.update({header_entry:i}) + + # review the first content row to confirm matching number of entries and test data type + + column_size = len(column_map.items()) + + if header_row: + test_row = output[1] + else: + test_row = output[0] + + # basic minimal check + if len(test_row) != column_size and not column_mapping_dict: + raise LLMWareException(message=f"Exception: Number of elements in first row of data - {len(test_row)} - " + f"does not match the number of column names in the column mapping - " + f"{column_size}.") + + if required_column_count: + if len(test_row) != required_column_count: + raise LLMWareException(message=f"Exception: Number of elements in first row of data - {len(test_row)} - " + f"does not match the number specified in required_column_count parameter - " + f"{required_column_count}.") + + column_map_inverted = {v: k for k, v in column_map.items()} + + for i, entry in enumerate(test_row): + + if i in column_map_inverted: + + if data_type_map: + + if column_map_inverted[i] in data_type_map: + dt = data_type_map[column_map_inverted[i]] + elif i in data_type_map: + dt = data_type_map[i] + else: + dt = self._get_best_guess_value_type(test_row[i]) + else: + dt = self._get_best_guess_value_type(test_row[i]) + + schema.update({column_map_inverted[i]:dt}) + + self.schema = schema + + if header_row: + starter = 1 + else: + starter = 0 + + # iterate thru all rows (excluding header_row, if any) + for x in range(starter,len(output)): + + new_row = {} + + if (len(output[x]) != column_size and not column_mapping_dict) or (required_column_count and + len(output[x] != required_column_count)): + logger.warning(f"warning: line - {x} - has {len(output[x])} elements, and does not match the " + f"number of columns in the schema - {column_size} - or specified in " + f"required_column_count - {required_column_count} - for safety, this row is being " + f"skipped.") + skipped_rows.append([x,output[x]]) + + else: + for i, entry in enumerate(output[x]): + + if i in column_map_inverted: + + # handle missing or null values + if not entry: + entry = "0" + new_row.update({column_map_inverted[i]: entry}) + + if len(new_row.items()) != column_size: + logger.warning(f"warning: on line - {x} - extracted elements - {len(new_row.items())} - does " + f"not map to the target column size - {column_size} - skipping row.") + else: + self.rows.append(new_row) + + output = {"rows": len(self.rows), "columns": column_size, "schema": self.schema, + "skipped_rows": []} + + return output + + def sql_table_create_string(self,table_name=None,schema=None): + + """ Builds a 'CREATE SQL TABLE' schema string that can be used directly as context in a Text2SQL model. + Optional parameters to pass a table_name and schema - if none provided, then it will pull from the + existing CustomTable parameters. + + Returns 'CREATE SQL TABLE' string as output. """ + + if schema: + self.schema= schema + + if table_name: + self.table_name = table_name + + sql_create_table = "" + + if not schema: + + try: + self.get_schema(self.table_name) + except: + raise LLMWareException(message=f"Exception: could not locate schema for selected " + f"table - {self.table_name}") + + if self.schema and self.table_name: + + keys_list = "(" + + sql_create_table = f"CREATE TABLE {self.table_name} (" + + for key, data_types in self.schema.items(): + + if key != "PRIMARY KEY": + + keys_list += key + ", " + sql_create_table += key + " " + data_types + ", " + + if sql_create_table.endswith(", "): + sql_create_table = sql_create_table[:-2] + + sql_create_table += " )" + + if keys_list.endswith(", "): + keys_list = keys_list[:-2] + + keys_list += " )" + + self.sql_create_table = sql_create_table + + return sql_create_table + + def insert_rows(self, table_name=None, rows=None, schema=None, callback=None): + + """ Designed for rapid prototyping - by default, intended to be called after a preliminary step of + 'load_csv' or 'load_json' file, which will package up the schema and save the rows, e.g., + + ct = CustomTable(table_name='my_test_table', db_name='sqlite') + analysis = ct.validate_csv('/path/to/csv', 'file.csv') + ct.load_csv('/path/to/csv', 'file.csv') + ct.insert_rows() + + If table_name and schema not previously passed/created, then these parameters should be passed in + calling this method. + + This method will insert the rows, held/packaged in self.rows, into the table.""" + + if table_name: + self.table_name = table_name + + if rows: + self.rows = rows + + if schema: + self.schema = schema + + rows_completed = 0 + + if self.build_table(): + + for i in range(0, len(self.rows)): + + if i >= 100 and i % 100 == 0: + + msg = f"update: CustomTable - insert_rows - status - rows loaded - " \ + f"{i} - out of {len(self.rows)}" + + if not callback: + logger.info(msg) + elif callback: + callback(msg) + + try: + self.write_new_record(self.rows[i]) + + except: + logger.warning(f"warning: write transaction not successful - skipping row - " + f"{i} - {self.rows[i]}") + + rows_completed += 1 + + else: + raise LLMWareException(message=f"Exception: unable to confirm build_table - table_name - " + f"{self.table_name} - schema - {self.schema}") + + return rows_completed + + def insert_rows_generator(self, table_name=None, rows=None, schema=None, callback=None,yield_updates=True): + + """ Intended to be called as a generator function that will yield periodic progress updates - + intended for ingesting larger CSV files - by default, intended to be called after a preliminary step of + 'load_csv' or 'load_json' file, which will package up the schema and save the rows, e.g., + + ct = CustomTable(table_name='my_test_table', db_name='sqlite') + analysis = ct.validate_csv('/path/to/csv', 'file.csv') + ct.load_csv('/path/to/csv', 'file.csv') + + for i in ct.insert_rows_generator(yield_updates=True, callback=callback_fn): + # insert logic to capture/update on progress of every 100 rows + + If table_name and schema not previously passed/created, then these parameters should be passed in + calling this method. + + This method will insert the rows, held/packaged in self.rows, into the table.""" + + if table_name: + self.table_name = table_name + + if rows: + self.rows = rows + + if schema: + self.schema = schema + + rows_completed = 0 + + if self.build_table(): + + for i in range(0, len(self.rows)): + + if i >= 100 and i % 100 == 0: + + msg = f"update: CustomTable - insert_rows - status - rows loaded - " \ + f"{i} - out of {len(self.rows)}" + + if not callback and not yield_updates: + logger.info(msg) + elif callback and not yield_updates: + callback(msg) + else: + yield i + + try: + self.write_new_record(self.rows[i]) + + except: + logger.warning(f"warning: write transaction not successful - skipping row - " + f"{i} - {self.rows[i]}") + + rows_completed += 1 + + else: + raise LLMWareException(message=f"Exception: unable to confirm build_table - table_name - " + f"{self.table_name} - schema - {self.schema}") + + return rows_completed + + def get_schema(self, table_name=None): + + """ Returns the schema for the table_name provided. If no table_name provided, then it will pull from + self.table_name """ + + if table_name: + self.table_name = table_name + + conn = self.get_connection(type="read") + schema = conn.get_schema(self.table_name) + conn.close() + + self.schema = schema + return schema + + def list_all_tables(self): + + """ Returns a list of all of the tables on the selected database. """ + + conn = self.get_connection(type="read") + all_tables = conn.list_all_tables() + conn.close() + + return all_tables + + def write_new_record(self, new_record, table_name=None): + + """ Writes a single new record to the database in the table provided, either directly (optional), or in + self.table_name. """ + + if table_name: + self.table_name = table_name + + conn = self.get_connection(type="write") + response = conn.write_new_record(new_record) + self.close_connection() + + return response + + def lookup(self, key, value, table_name=None): + + """ Core lookup method takes a 'key' and 'value' as input, and performs a lookup on the target DB and table + and returns the corresponding full row as a python dictionary with keys corresponding to the schema. """ + + if table_name: + self.table = table_name + + conn = self.get_connection(type="read") + response = conn.lookup(key, value) + + return response + + def custom_lookup(self, custom_filter): + + """ Input is a custom filter which will be applied directly to the database. + + For MongoDB, this should be a filter dictionary, following standard Mongo query structures. + + For Postgres and SQLite, this should be a SQL string - which will be passed directly to the DB + without modification or safety checks. """ + + # pass filter/params directly to resource + conn = self.get_connection(type="read") + + try: + response = conn.direct_custom_query(custom_filter) + except: + logger.warning(f"warning: query was not successful - {str(custom_filter)} - and generated an error " + f"when attempting to run on table - {self.table_name} - database - {self.db}. ") + response = [] + + return response + + def get_all(self, table_name=None, return_cursor=False): + + """ Returns all rows from the DB table as a list of python dictionaries, corresponding to the schema - + assumed to be able to fit in memory. This method should not be used for extremely large tables + where a cursor/iterator should be used. """ + + if table_name: + self.table_name = table_name + + conn = self.get_connection(type="read") + response_cursor = conn.get_whole_collection() + + # in most cases, it is more convenient to exhaust the cursor and pull the whole set of results into + # memory. + # TODO: implement cursor option - see DBCursor class for more details. + + if not return_cursor: + output = response_cursor.pull_all() + else: + output = response_cursor.pull_all() + + conn.close() + + return output + + def get_distinct_list(self, key, table_name=None): + + """ Returns the distinct elements for a selected key in the database table. """ + + if table_name: + self.table_name = table_name + + conn = self.get_connection(type="read") + response = conn.get_distinct_list(key) + conn.close() + + return response + + def count_documents(self, filter_dict): + + conn = self.get_connection(type="read") + count = conn.count_documents(filter_dict) + conn.close() + + return count + + def delete_record(self, key, value, table_name=None): + + """ Deletes a selected record(s) with matching key:value in selected table. """ + + if table_name: + self.table_name = table_name + + conn = self.get_connection(type="write") + d = conn.delete_record_by_key(key, value) + + conn.close() + + return 0 + + def update_record(self, filter_dict, key, new_value): + + """ Updates a selected record(s) identified with filter_dict {any_key:any_value}, and then sets the new_value + of the attribute value for key in that record. """ + + conn = self.get_connection(type="write") + confirmation = conn.update_one_record(filter_dict, key, new_value) + conn.close() + + return 0 + + def delete_table(self, table_name=None, confirm=False): + + """ Deletes a selected table in the database - note: the optional 'confirm' parameter must be set to + True explicitly as a safety check. """ + + if table_name: + self.table_name = table_name + + conn = self.get_connection(type="write") + confirmation = conn.destroy_collection(confirm_destroy=confirm) + + conn.close() + + return 0 + + +class CloudBucketManager: + + """Main class for handling basic operations with Cloud Buckets - specifically for AWS S3""" + + def __init__(self): + # placeholder - no state / config required currently + self.s3_access_key = AWSS3Config().get_config("access_key") + self.s3_secret_key = AWSS3Config().get_config("secret_key") + + # used in Setup() to get sample test documents + def pull_file_from_public_s3(self, object_key, local_file, bucket_name): + + """Pulls selected file from public S3 bucket - used by Setup to get sample files""" + + # return list of successfully downloaded files + downloaded_files = [] + + try: + # Ensure the local file's folder exists + os.makedirs(os.path.dirname(local_file), exist_ok=True) + + s3 = boto3.resource('s3', config=Config(signature_version=UNSIGNED)) + s3.Bucket(bucket_name).download_file(object_key, local_file) + downloaded_files.append(object_key) + + except ClientError as e: + logger.error(e) + + return downloaded_files + + def create_local_model_repo(self, access_key=None,secret_key=None): + + """Pulls down and caches models from public llmware public S3 repo """ + + # list of models retrieved from cloud repo + models_retrieved = [] + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + # confirm that local model repo path has been created + local_model_repo_path = LLMWareConfig.get_model_repo_path() + if not os.path.exists(local_model_repo_path): + os.mkdir(local_model_repo_path) + + aws_repo_bucket = LLMWareConfig.get_config("llmware_public_models_bucket") + + # if specific model_list passed, then only retrieve models on the list + + bucket = boto3.resource('s3', aws_access_key_id=access_key, + aws_secret_access_key=secret_key).Bucket(aws_repo_bucket) + + files = bucket.objects.all() + + s3 = boto3.client('s3', aws_access_key_id=access_key, aws_secret_access_key=secret_key) + + # bucket = s3.Bucket(bucket_name) + # files = bucket.objects.all() + + for file in files: + + name_parts = file.key.split(os.sep) + + # confirm that file.key is correctly structure as [0] model name, and [1] model component + if len(name_parts) == 2: + + logger.info(f"update: identified models in model_repo: {name_parts}") + + if name_parts[0] and name_parts[1]: + + model_folder = os.path.join(local_model_repo_path,name_parts[0]) + + if not os.path.exists(model_folder): + os.mkdir(model_folder) + models_retrieved.append(name_parts[0]) + + logger.info(f"update: downloading file from s3 bucket - " + f"{name_parts[1]} - {file.key}") + + s3.download_file(aws_repo_bucket, file.key, os.path.join(model_folder,name_parts[1])) + + logger.info(f"update: created local model repository - {len(models_retrieved)} models retrieved - " + f" model list - {models_retrieved}") + + return models_retrieved + + def pull_single_model_from_llmware_public_repo(self, model_name=None): + + """Pulls selected model from llmware public S3 repo to local model repo""" + + # if no model name selected, then get all + bucket_name = LLMWareConfig().get_config("llmware_public_models_bucket") + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig().setup_llmware_workspace() + + model_path_local = LLMWareConfig.get_model_repo_path() + + if not os.path.exists(model_path_local): + os.makedirs(model_path_local) + + # assumes that files in model folder are something like: + # "pytorch_model.bin" | "config.json" | "tokenizer.json" + + bucket = boto3.resource('s3', config=Config(signature_version=UNSIGNED)).Bucket(bucket_name) + + files = bucket.objects.all() + + for file in files: + + if file.key.startswith(model_name): + + # found component of model in repo, so go ahead and create local model folder, if needed + local_model_folder = os.path.join(model_path_local, model_name) + if not os.path.exists(local_model_folder): + os.mkdir(local_model_folder) + + # simple model_repo structure - each model is a separate folder + # each model is a 'flat list' of files, so safe to split on ("/") to get key name + if not file.key.endswith('/'): + local_file_path = os.path.join(local_model_folder,file.key.split('/')[-1]) + bucket.download_file(file.key, local_file_path) + + logger.info(f"update: successfully downloaded model - {model_name} - " + f"from aws s3 bucket for future access") + + return files + + # called in Library as convenience method to connect to user S3 bucket and download into library path + def connect_to_user_s3_bucket (self, aws_access_key, aws_secret_key, + user_bucket_name, local_download_path, max_files=1000): + + """Connects to user S3 bucket""" + + files_copied = [] + + accepted_file_formats = ["pptx", "xlsx", "docx", "pdf", "txt", "csv", "html", "jsonl", + "jpg", "jpeg", "png", "wav", "zip"] + + try: + s3 = boto3.client('s3', aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key) + + bucket = boto3.resource('s3', aws_access_key_id=aws_access_key, + aws_secret_access_key=aws_secret_key).Bucket(user_bucket_name) + + files = bucket.objects.all() + + for file in files: + + # strip os.sep from file name + safe_file_name = str(file.key) + if safe_file_name.startswith(os.sep): + safe_file_name = safe_file_name[1:] + + f = safe_file_name.replace(os.sep, "_") + f = f.replace(" ", "_") + + file_type = f.split(".")[-1].lower() + if file_type in accepted_file_formats: + s3.download_file(user_bucket_name, file.key, local_download_path + f) + files_copied.append(f) + if len(files_copied) > max_files: + break + + except: + logger.error(f"error: could not connect to s3 bucket - {user_bucket_name}") + + return files_copied + + return files_copied + + def upload_file(self,file_name, bucket, object_name=None, aws_secret_key=None, aws_access_key=None): + + """Upload a file to an S3 bucket + + :param file_name: File to upload + :param bucket: Bucket to upload to + :param object_name: S3 object name. If not specified then file_name is used + :return: True if file was uploaded, else False + """ + + # If S3 object_name was not specified, use file_name + if object_name is None: + object_name = os.path.basename(file_name) + + # Upload the file + s3_client = boto3.client('s3',aws_access_key_id=aws_access_key,aws_secret_access_key=aws_secret_key) + try: + response = s3_client.upload_file(file_name, bucket, object_name) + except ClientError as e: + logging.error(e) + return False + return True + + def fetch_model_from_bucket(self, model_name=None, bucket_name=None, save_path=None): + + """Pulls selected model from llmware public S3 repo to local model repo""" + + # if no model name selected, then get all + if not bucket_name: + bucket_name = LLMWareConfig().get_config("llmware_public_models_bucket") + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig().setup_llmware_workspace() + + if not save_path: + save_path = LLMWareConfig.get_model_repo_path() + + if not os.path.exists(save_path): + os.makedirs(save_path) + + # assumes that files in model folder are something like: + # "pytorch_model.bin" | "config.json" | "tokenizer.json" + + bucket = boto3.resource('s3', config=Config(signature_version=UNSIGNED)).Bucket(bucket_name) + + files = bucket.objects.all() + + for file in files: + + if file.key.startswith(model_name): + + # found component of model in repo, so go ahead and create local model folder, if needed + local_model_folder = os.path.join(save_path, model_name) + if not os.path.exists(local_model_folder): + os.mkdir(local_model_folder) + + # simple model_repo structure - each model is a separate folder + # each model is a 'flat list' of files, so safe to split on ("/") to get key name + if not file.key.endswith('/'): + local_file_path = os.path.join(local_model_folder, file.key.split('/')[-1]) + bucket.download_file(file.key, local_file_path) + + logger.info(f"update: successfully downloaded model - {model_name} - " + f"from aws s3 bucket for future access") + + return files + + +class ParserState: + + """ ParserState is the main class abstraction to manage and persist Parser State """ + + def __init__(self, parsing_output=None, parse_job_id=None): + + self.parse_job_id = parse_job_id + self.parsing_output = parsing_output + self.parser_job_output_base_name = "parser_job_" + self.parser_output_format = ".jsonl" + self.parser_output_fp = LLMWareConfig.get_parser_path() + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + def get_parser_state_fn_from_id(self, parser_job_id): + + """ Generates the state filename from parser_job_id """ + + fn = self.parser_job_output_base_name + str(parser_job_id) + self.parser_output_format + return fn + + def get_parser_id_from_parser_state_fn(self, fn): + + """ Returns the parser id extracted from the parser state filename """ + + core_fn = fn.split(".")[0] + id = core_fn.split("_")[-1] + return id + + def lookup_by_parser_job_id(self, parser_id): + + """ Look up the parser job id""" + + parser_output = self.lookup_by_parse_job_id(parser_id) + return parser_output + + def save_parser_output(self, parser_job_id, parser_output): + + """ Saves the parser output to jsonl file in parser history """ + + fn = self.get_parser_state_fn_from_id(parser_job_id) + fp = os.path.join(self.parser_output_fp, fn) + + outfile = open(fp, "w", encoding='utf-8') + + for entry_dict in parser_output: + jsonl_row = json.dumps(entry_dict) + outfile.write(jsonl_row) + outfile.write("\n") + + outfile.close() + + return fn + + def issue_new_parse_job_id(self, custom_id=None, mode="uuid"): + + """ Issues new parse_job_id """ + + if custom_id: + self.parse_job_id = custom_id + else: + if mode == "time_stamp": + self.parse_job_id = StateResourceUtil().get_current_time_now() + elif mode == "uuid": + self.parse_job_id = str(StateResourceUtil().get_uuid()) + elif mode == "random_number": + self.parse_job_id = str(random.randint(1000000, 9999999)) + + return self.parse_job_id + + def lookup_by_parse_job_id (self, prompt_id): + + """ Lookup by parse_job_id """ + + output = [] + + fn = self.get_parser_state_fn_from_id(prompt_id) + fp = os.path.join(self.parser_output_fp, fn) + + try: + my_file = open(fp, 'r', encoding='utf-8') + for lines in my_file: + new_row = json.loads(lines) + output.append(new_row) + + except: + logger.warning(f"update: ParserState - could not find previous parse job record - {prompt_id}") + output = [] + + return output + + +class PromptState: + + """ PromptState is the main class abstraction that handles persisting and lookup of Prompt interactions""" + + def __init__(self, prompt=None): + + self.prompt = prompt + self.prompt_state_base_name = "prompt_" + self.prompt_state_format = ".jsonl" + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + self.prompt_path = LLMWareConfig.get_prompt_path() + self.output_path = LLMWareConfig.get_prompt_path() + + # edge case - if llmware main path exists, but prompt path not created or deleted + if not os.path.exists(self.prompt_path): + os.mkdir(self.prompt_path) + os.chmod(self.prompt_path, 0o777) + + # prompt state written to files + self.prompt_collection = None + self.write_to_db = False + + def get_prompt_state_fn_from_id(self, prompt_id): + + """Generates the prompt state filename from prompt_id """ + fn = self.prompt_state_base_name + str(prompt_id) + self.prompt_state_format + return fn + + def get_prompt_id_from_prompt_state_fn(self, fn): + + """ Gets the prompt id extracted from prompt state filename """ + + core_fn = fn.split(".")[0] + id = core_fn.split("_")[-1] + return id + + def lookup_by_prompt_id(self, prompt_id): + + """ Lookup by prompt id to retrieve a persisted prompt interaction """ + + ai_trx_list = self.lookup_by_prompt_id_from_file(prompt_id) + return ai_trx_list + + def register_interaction(self, ai_dict): + + """ Registers a new prompt interaction into the interaction history in memory """ + + # by default, add to the interaction_history in memory + self.prompt.interaction_history.append(ai_dict) + + return ai_dict + + def initiate_new_state_session(self, prompt_id=None): + + """ Starts a new state session for an indefinite set of prompt interactions """ + + if not prompt_id: + prompt_id = self.issue_new_prompt_id() + + # reset key trackers + self.prompt.llm_history = [] + self.prompt.prompt_id = prompt_id + return prompt_id + + def issue_new_prompt_id(self, custom_id=None, mode="uuid"): + + """ Issues a new prompt id """ + + # issue new prompt_id + if custom_id: + self.prompt.prompt_id = custom_id + else: + if mode == "time_stamp": + self.prompt.prompt_id = StateResourceUtil().get_current_time_now() + elif mode == "uuid": + self.prompt.prompt_id = str(StateResourceUtil().get_uuid()) + elif mode == "random_number": + self.prompt.prompt_id = str(random.randint(1000000, 9999999)) + + return self.prompt.prompt_id + + def load_state(self, prompt_id, prompt_path=None,clear_current_state=True): + + """ Loads a saved prompt interaction history from disk """ + + output = None + + if not prompt_path: + prompt_path = self.prompt_path + + fn = self.get_prompt_state_fn_from_id(prompt_id) + fp = os.path.join(prompt_path, fn) + + try: + if clear_current_state: + self.prompt.interaction_history = [] + + my_file = open(fp, 'r', encoding='utf-8') + for lines in my_file: + new_row = json.loads(lines) + self.prompt.interaction_history.append(new_row) + self.prompt.prompt_id = prompt_id + output = self.prompt.interaction_history + + except: + logger.warning(f"update: PromptState - could not find previous prompt interaction state- {prompt_id}") + output = None + + return output + + def lookup_by_prompt_id_from_file(self, prompt_id): + + """ Lookup prompt id from file """ + + output = [] + + fn = self.get_prompt_state_fn_from_id(prompt_id) + fp = os.path.join(self.prompt_path, fn) + + try: + my_file = open(fp, 'r', encoding='utf-8') + for lines in my_file: + new_row = json.loads(lines) + output.append(new_row) + except: + logger.warning(f"warning: PromptState - could not find previous prompt interaction state- {prompt_id}") + output = [] + + return output + + def full_history(self): + + """ Returns the full prompt history from disk """ + + ai_trx_list = self.full_history_from_file() + return ai_trx_list + + def full_history_from_file(self): + + """ Returns the full prompt history from disk """ + + output= [] + + all_prompts = os.listdir(self.prompt_path) + + for i, files in enumerate(all_prompts): + + # will iterate through all files in the prompt path that start with the expected + # prompt base name + + if files.startswith(self.prompt_state_base_name): + prompt_id = self.get_prompt_id_from_prompt_state_fn(files) + records = self.lookup_by_prompt_id(prompt_id) + output += records + + return output + + def lookup_prompt_with_filter(self, filter_dict): + + """ Enables lookup of prompt history with filter """ + + # default - return [] + output = [] + + # may want to add safety check that filter_dict is dict + all_prompt_records = self.full_history_from_file() + + for i, prompt in enumerate(all_prompt_records): + match = -1 + for keys, values in filter_dict.items(): + + # must match every key in the filter dict + if keys in prompt: + if values == prompt[keys]: + match = 1 + else: + match = -1 + break + else: + # if key not in record, then not a match + match = -1 + break + + if match == 1: + output.append(prompt) + + return output + + def update_records(self, prompt_id, filter_dict, update_dict): + + """ Enables update of a prompt interaction history from file """ + + updated_prompt_records = [] + matching_record = {} + prompt_records = self.lookup_by_prompt_id(prompt_id) + for record in prompt_records: + match = -1 + for keys, values in filter_dict.items(): + if keys in record: + if record[keys] == values: + match = 1 + else: + match = -1 + break + else: + match = -1 + break + + if match == -1: + updated_prompt_records.append(record) + else: + # found matching record + matching_record = record + + # update records according to update_dict + updated_record = {} + for key, value in matching_record.items(): + for update_key, update_value in update_dict.items(): + if key == update_key: + updated_record.update({key: update_value}) + else: + updated_record.update({key:value}) + + updated_prompt_records.append(updated_record) + + self.save_custom_state(prompt_id, updated_prompt_records) + + return 0 + + def save_custom_state(self, prompt_id, custom_history, prompt_path=None): + + """ Saves state """ + + if not prompt_path: + prompt_path = LLMWareConfig.get_prompt_path() + + fn = self.get_prompt_state_fn_from_id(prompt_id) + fp = os.path.join(prompt_path, fn) + + outfile = open(fp, "w", encoding='utf-8') + + for entry_dict in custom_history: + jsonl_row = json.dumps(entry_dict) + outfile.write(jsonl_row) + outfile.write("\n") + + outfile.close() + + return fp + + def save_state(self, prompt_id, prompt_path=None): + + """ Saves state """ + + if not prompt_path: + prompt_path = LLMWareConfig.get_prompt_path() + + fn = self.get_prompt_state_fn_from_id(prompt_id) + fp = os.path.join(prompt_path, fn) + + outfile = open(fp, "w", encoding='utf-8') + + for entry_dict in self.prompt.interaction_history: + jsonl_row = json.dumps(entry_dict) + outfile.write(jsonl_row) + outfile.write("\n") + + outfile.close() + + return fp + + def available_states(self, prompt_path=None): + + """ Lists available prompt interaction history states """ + + available_states = [] + + if not prompt_path: + prompt_path = self.prompt_path + + for x in os.listdir(prompt_path): + if x.startswith(self.prompt_state_base_name): + prompt_id = self.get_prompt_id_from_prompt_state_fn(x) + new_entry = {"prompt_id": prompt_id, "prompt_fn": x} + available_states.append(new_entry) + + logger.info(f"update: PromptState - available states - {available_states}") + + return available_states + + def generate_interaction_report(self, prompt_id_list, output_path=None, report_name=None): + + """ Prepares a csv report that can be extracted to a spreadsheet """ + + if not output_path: + output_path = self.output_path + + if not report_name: + report_name = "interaction_report_" + str(StateResourceUtil().get_current_time_now()) + ".csv" + + result_count = 0 + + report_fp = os.path.join(output_path,report_name) + + with open(report_fp, 'w', encoding='utf-8', newline='') as csvfile: + c = csv.writer(csvfile, dialect='excel', doublequote=False, delimiter=',', escapechar=']') + + header_row = ["Prompt_ID", "Prompt", "LLM_Response", "Instruction", "Evidence", "Model", "Time-Stamp"] + c.writerow(header_row) + + for i, prompt_id in enumerate(prompt_id_list): + + fn = self.get_prompt_state_fn_from_id(prompt_id) + fp = os.path.join(self.prompt_path, fn) + + my_file = open(fp, 'r', encoding='utf-8') + for lines in my_file: + new_row = json.loads(lines) + + # create new csv row + # strip symbols that can disrupt csv output + evidence_clean = re.sub(r"[,\"\'\n\r\u2018\u2019\u201c\u201d]"," ", new_row['evidence']) + response_clean = re.sub(r"[,\"\'\n\r\u2018\u2019\u201c\u201d]", " ", new_row['llm_response']) + + csv_row = [prompt_id, + new_row["prompt"], + response_clean, + new_row["instruction"], + evidence_clean, + new_row["model"], + new_row["time_stamp"] + ] + + c.writerow(csv_row) + result_count += 1 + + csvfile.close() + + output_response = {"report_name": report_name, "report_fp": report_fp, "results": result_count} + + return output_response + + def generate_interaction_report_current_state(self, output_path=None, report_name=None): + + """ Prepares a csv report that can be extracted to a spreadsheet """ + + if not output_path: + output_path = self.output_path + + if not report_name: + report_name = "interaction_report_" + str(StateResourceUtil().get_current_time_now()) + ".csv" + + result_count = 0 + + report_fp = os.path.join(output_path,report_name) + + with open(report_fp, 'w', encoding='utf-8', newline='') as csvfile: + c = csv.writer(csvfile, dialect='excel', doublequote=False, delimiter=',', escapechar=']') + + header_row = ["Prompt_ID", "Prompt", "LLM_Response", "Instruction", "Evidence", "Model", "Time-Stamp"] + c.writerow(header_row) + + for i, new_row in enumerate(self.prompt.interaction_history): + + # create new csv row + # strip symbols that can disrupt csv output + evidence_clean = re.sub(r"[,\"\'\n\r\u2018\u2019\u201c\u201d]", " ", new_row['evidence']) + response_clean = re.sub(r"[,\"\'\n\r\u2018\u2019\u201c\u201d]", " ", new_row['llm_response']) + + csv_row = [self.prompt.prompt_id, + new_row["prompt"], + response_clean, + new_row["instruction"], + evidence_clean, + new_row["model"], + new_row["time_stamp"] + ] + + c.writerow(csv_row) + result_count += 1 + + csvfile.close() + + output_response = {"report_name": report_name, "report_fp": report_fp, "results": result_count} + + return output_response + + +class QueryState: + + """ QueryState is the main class abstraction to manage persistence of Queries """ + + def __init__(self, query=None, query_id=None): + + if query: + self.query = query + self.query_id = query.query_id + + if query_id: + self.query_id = query_id + self.query = None + + self.query_state_base_name = "query_" + self.query_state_format = ".jsonl" + + self.query_path = LLMWareConfig.get_query_path() + self.output_path = LLMWareConfig.get_query_path() + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + # if llmware main path exists, but query_path not created or deleted + if not os.path.exists(self.query_path): + os.mkdir(self.query_path) + os.chmod(self.query_path, 0o777) + + def get_query_state_fn_from_id(self, prompt_id): + + """ Generates query state filename from query id """ + + fn = self.query_state_base_name + str(prompt_id) + self.query_state_format + return fn + + def get_query_id_from_prompt_state_fn(self, fn): + + """ Extracts the query id from the filename """ + + core_fn = fn.split(".")[0] + id = core_fn.split("_")[-1] + return id + + def initiate_new_state_session(self, query_id=None): + + """ Starts a new state session in memory - tracks all query results and metadata in session """ + + if not query_id: + query_id = self.issue_new_query_id() + + # reset key trackers + self.query.query_history = [] + self.query.results = [] + self.query.doc_id_list = [] + self.query.doc_fn_list = [] + + self.query_id = query_id + + return query_id + + def issue_new_query_id(self, custom_id=None, mode="uuid"): + + """ Issue new query_id """ + + if not custom_id: + + if mode == "time_stamp": + custom_id = StateResourceUtil().get_current_time_now() + elif mode == "uuid": + custom_id = StateResourceUtil().get_uuid() + elif mode == "random_number": + custom_id = str(random.randint(1000000, 9999999)) + + return custom_id + + def available_states(self): + + """ Gets all available saved query states on file """ + + available_states = [] + + for x in os.listdir(self.query_path): + if x.startswith(self.query_state_base_name): + query_id = self.get_query_id_from_prompt_state_fn(x) + new_entry = {"query_id": query_id, "query_fn": x} + available_states.append(new_entry) + + logger.info(f"update: QueryState - available saved query states - {available_states}") + + return available_states + + def load_state (self, query_id): + + """ Loads query state from file """ + + output = [] + doc_id_list = [] + doc_fn_list = [] + query_history = [] + + fn = self.get_query_state_fn_from_id(query_id) + fp = os.path.join(self.query_path, fn) + + try: + my_file = open(fp, 'r', encoding='utf-8') + for lines in my_file: + new_row = json.loads(lines) + output.append(new_row) + + if "doc_ID" in new_row: + if new_row["doc_ID"] not in doc_id_list: + doc_id_list.append(new_row["doc_ID"]) + + if "file_source" in new_row: + if new_row["file_source"] not in doc_fn_list: + doc_fn_list.append(new_row["file_source"]) + + if "query" in new_row: + if new_row["query"] not in query_history: + query_history.append(new_row["query"]) + + except: + logger.warning(f"update: QueryState - could not find previous query state- {query_id}") + output = [] + + self.query.results = output + self.query.doc_id_list = doc_id_list + self.query.doc_fn_list = doc_fn_list + self.query.query_history = query_history + + return self + + def save_state(self, query_id=None): + + """ Saves query state to jsonl file in query state history """ + + if not query_id: + query_id = self.query.query_id + + fn = self.get_query_state_fn_from_id(query_id) + fp = os.path.join(self.query_path, fn) + + outfile = open(fp, "w", encoding='utf-8') + + for entry_dict in self.query.results: + jsonl_row = json.dumps(entry_dict) + outfile.write(jsonl_row) + outfile.write("\n") + + outfile.close() + + return fn + + def generate_query_report_current_state(self, report_name=None): + + """ Prepares a csv report that can be extracted to a spreadsheet """ + + if not self.query: + logger.error("error: QueryState - report generation - must load a current query") + return -1 + + query_name = "" + for entries in self.query.query_history: + query_name += re.sub(" ", "_", entries) + "-" + + if len(query_name) > 100: + query_name = query_name[0:100] + if query_name.endswith("-"): + query_name = query_name[:-1] + + if not report_name: + report_name = "query_report_" + query_name + ".csv" + + report_out = [] + col_headers = ["Query", "File_Source", "Doc_ID", "Block_ID", "Page", "Text"] + + report_out.append(col_headers) + + for j, results in enumerate(self.query.results): + + query = "" + if "query" in results: + query = results["query"] + + file_source = "" + if "file_source" in results: + file_source = results["file_source"] + + doc_id = "NA" + if "doc_ID" in results: + doc_id = results["doc_ID"] + + block_id = "NA" + if "block_ID" in results: + block_id = results["block_ID"] + + page_num = "NA" + if "page_num" in results: + page_num = results["page_num"] + + text = "" + if "text" in results: + text = re.sub(r"[,\"]"," ", results["text"]) + + new_row = [query, file_source, doc_id, block_id, page_num, text] + + report_out.append(new_row) + + fp = os.path.join(self.query_path, report_name) + + StateResourceUtil().file_save(report_out, self.output_path, report_name) + + return report_name + + +class StateResourceUtil: + + """ Utility methods for the State Resource classes """ + + def __init__(self): + self.do_nothing = 0 # placeholder - may add attributes in the future + + def get_uuid(self): + """ Uses unique id creator from uuid library """ + return uuid.uuid4() + + @staticmethod + def get_current_time_now (time_str="%a %b %e %H:%M:%S %Y"): + """ Gets current time """ + + # if time stamp used in filename, needs to be Windows compliant + if platform.system() == "Windows": + time_str = "%Y-%m-%d_%H%M%S" + + return datetime.now().strftime(time_str) + + @staticmethod + def file_save (cfile, file_path, file_name): + + """ Saves list/array to csv file to disk """ + + max_csv_size = 20000 + csv.field_size_limit(max_csv_size) + + out_file = file_path + file_name + with open(out_file, 'w', newline='') as csvfile: + c = csv.writer(csvfile, dialect='excel', doublequote=False, delimiter=',', escapechar= ']') + # c.writerow(first_row) + for z in range(0 ,len(cfile)): + # intercept a line too large here + if sys.getsizeof(cfile[z]) < max_csv_size: + c.writerow(cfile[z]) + else: + logger.error(f"error: CSV ERROR: Row exceeds MAX SIZE: " + f"{sys.getsizeof(cfile[z])} - {cfile[z]}") + + csvfile.close() + + return 0 + + +class Status: + + """Provides callers with an interface on the status of the parsing and embedding process. + + ``Status`` is the central class for accessing (reading and writing) the status of processes. + The intended use case is to be an interface for non-llmware components (the callers) that need + information on llmware progress, e.g user interface components may need to change depending on the + progress of parsing. A status consists of a summary string and metrics that can be used to provide + graphical widgets an update. If a status is written to SQL collection database, then it will use the + Status schema defined in configs.py. + + Parameters + ---------- + account_name : str, optional, default='llmware' + Sets the name of the account, which is used for writting and retrieving a status. + + Returns + ------- + status : Status + A new ``Status`` object. + + """ + + def __init__(self, account_name="llmware"): + + self.account_name = account_name + self.schema = LLMWareTableSchema.get_status_schema() + + # if table does not exist (and required by the underlying collection db), then create + if CollectionWriter("status", account_name=self.account_name).check_if_table_build_required(): + # create "status" table + CollectionWriter("status", account_name=self.account_name).create_table("status", self.schema) + + def get_pdf_parsing_status(self, library_name, job_id="0"): + + """ Gets the status written by the PDF parser """ + + status_key = f"{library_name}_pdf_parser_{job_id}" + status = CollectionRetrieval("status", account_name=self.account_name).lookup("key", status_key) + + return status + + def get_office_parsing_status(self, library_name, job_id="0"): + + """ Gets the status written by the Office parser """ + + status_key = f"{library_name}_office_parser_{job_id}" + status = CollectionRetrieval("status", account_name=self.account_name).lookup("key", status_key) + + return status + + # Return the status dict + def get_embedding_status(self, library_name, embedding_model): + + """ Gets the embedding status written by the EmbeddingHandler class and each supported vector DB """ + + status_key = self._get_embedding_status_key(library_name, embedding_model) + status = CollectionRetrieval("status", account_name=self.account_name).lookup("key", status_key) + + return status + + def new_embedding_status(self, library_name, embedding_model, total): + + """ Creates a new embedding status - invoked at start of embedding job """ + + status_key = self._get_embedding_status_key(library_name, embedding_model) + status_entry = { + "key": status_key, + "summary": f"0 of {total} blocks", + "start_time": time.time(), + "end_time": None, + "total": total, + "current": 0, + "units": "blocks" + } + CollectionWriter("status", account_name=self.account_name).replace_record({"key": status_key}, status_entry) + + return 0 + + def increment_embedding_status(self, library_name, embedding_model, progress): + + """ Increments the embedding status throughout the embedding job - enables parallelized writing and updates """ + + status_key = self._get_embedding_status_key(library_name, embedding_model) + + status_entry = CollectionRetrieval("status", account_name=self.account_name).lookup("key", status_key) + + if len(status_entry) == 1: + status_entry = status_entry[0] + + status_entry["current"] = status_entry["current"] + progress + if status_entry["current"] >= status_entry["total"]: + status_entry["end_time"] = time.time() + + status_entry["summary"] = f"{status_entry['current']} of {status_entry['total']} {status_entry['units']}" + + CollectionWriter("status", account_name=self.account_name).replace_record({"key": status_key}, status_entry) + + return 0 + + def tail_embedding_status(self, library_name, model_name, poll_seconds=0.2): + + """ Can be invoked in tests to poll and check and print out embedding status """ + + thread = Thread(target=self._tail_embedding_status, args=(library_name, model_name, poll_seconds)) + thread.daemon = True + thread.start() + + def _tail_embedding_status(self, library_name, model_name, poll_seconds=0.2): + + """ Display of embedding status """ + + current_summary = "" + while True: + status_dict = self.get_embedding_status(library_name, model_name) + if status_dict: + if current_summary != status_dict["summary"]: # If the status has changed, print it + current_summary = status_dict["summary"] + print(current_summary) + if status_dict["current"] >= status_dict["total"]: # If the job is done exit + return + time.sleep(poll_seconds) + + # Generate and return a unique key for status, combining the library_name and embedding_model + def _get_embedding_status_key(self, library_name, embedding_model): + + """ Gets the embedding status key """ + + return f"{library_name}_embedding_{embedding_model}" + diff --git a/llmware/retrieval.py b/llmware/retrieval.py new file mode 100755 index 0000000..c7b8c37 --- /dev/null +++ b/llmware/retrieval.py @@ -0,0 +1,1764 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The retrieval module implements the Query class. The Query class provides a high-level interface for executing +a variety of queries on a Library collection, whether instantiated on Mongo, Postgres, or SQLite. + +The Query class includes both text retrieval strategies, which operate directly as queries on the text collection +database, as well as vector embedding semantic retrieval strategies, which require the use of o vector DB and that the +embeddings were previously created for the Library. There are also a number of convenience methods that provide +'hybrid' strategies combining elements of semantic and text querying.""" + + +import logging +import os +from collections import Counter +from datetime import datetime + +try: + from bson.objectid import ObjectId +except: + pass + +from llmware.configs import LLMWareConfig, LLMWareException, ModelNotFoundException +from llmware.embeddings import EmbeddingHandler +from llmware.resources import CollectionRetrieval, QueryState +from llmware.util import Utilities, CorpTokenizer +from llmware.models import ModelCatalog + +logger = logging.getLogger(__name__) + + +class Query: + + """Implements the query capabilities against a ``Library` object`. + + Query is responsible for executing queries against an indexed library. The library can be semantic, text, custom, + or hybrid. A query object requires a library object as input, which will be the source of the query. + + Parameters + ---------- + library : Library object + A ``library`` object. + + embedding_model : object, default=None + An ``embedding_model`` object. + + tokenizer : object, default=None + + vector_db_api_key : str, default=None + The API key for the vector store. + + query_id : int, default=None + The identifier for a query. This is used when a query state has to be loaded. + + from_hf : bool, default=False + Sets whether the embedding model should be loaded from hugging face. + + from_sentence_transformer: bool, default=False + Sets whether the embedding model should be loaded from ``LLMWareSemanticModel``. + + embedding_model_name : str, default=None + The name of the embedding model. This has to be set if ``from_sentence_transformer=True``. + + save_history : bool, default=True + Sets whether the history of queries should be saved. + + query_mode : str, default=None + Sets the query mode that should be used. It has to be either 'text', 'semantic', or 'hybrid'. + + vector_db : str, default=None + The name of the vector store to be queried against. If it is not set, then this is determined by the + given ``embedding_model``. + + + Examples + ---------- + >>> from llmware.library import Library + >>> from llmware.retrieval import Query + >>> library = Library().create_new_library('lib_semantic_query') + >>> library.add_website(url='https://en.wikipedia.org/wiki/Austria', get_links=False) + >>> library.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="milvus", batch_size=500) + >>> query = Query(library=library) + >>> results = query.semantic_query(query='the capital of austria is', result_count=3) + >>> len(results) + 3 + >>> results[0].keys() + dict_keys(['query', '_id', 'text', 'doc_ID', 'block_ID', 'page_num', 'content_type', + 'author_or_speaker', 'special_field1', 'file_source', 'added_to_collection', + 'table', 'coords_x', 'coords_y', 'coords_cx', 'coords_cy', 'external_files', + 'score', 'similarity', 'distance', 'matches', 'account_name', 'library_name']) + >>> results[0]['query'] + 'the capital of austria is' + >>> results[0]['text'] + 'Austria is a parliamentary representative democracy with a popularly elected president as head of ' + 'state and a chancellor as head of government and chief executive. Major cities include Vienna , Graz, ' + 'Linz , Salzburg , and Innsbruck . Austria has the 17th highest nominal GDP per capita with high ' + 'standards of living; it was ranked 25th in the world for its Human Development Index in 2021. ' + >>> results[2]['text'] + "Austrian Parliament Building Vienna The Parliament of Austria is located in Vienna , the country's capital " + "and most populous city. Austria became a federal , representative democratic republic through the " + "Federal Constitutional Law of 1920. The political system of the Second Republic with its nine federal " + "states is based on the constitution of 1920, amended in 1929, which was re-enacted on 1 May 1945. [108] " + """ + + def __init__(self, library, embedding_model=None, tokenizer=None, vector_db_api_key=None, + query_id=None, from_hf=False, from_sentence_transformer=False,embedding_model_name=None, + save_history=True, query_mode=None, vector_db=None, model_api_key=None): + + # load user profile & instantiate core library assets linked to profile + + self.library = library + + if library: + self.library_name = library.library_name + self.account_name = library.account_name + else: + # throw error if library object does not have library_name and account_name attributes + raise LLMWareException(message= f"Query - init - library object not found - {library}") + + # explicitly pass name of embedding model, if multiple embeddings on library + self.embedding_model_name = embedding_model_name + + # added option to pass embedding_model and tokenizer + self.user_passed_model = embedding_model + self.user_passed_tokenizer = tokenizer + self.from_hf = from_hf + self.from_sentence_transformer = from_sentence_transformer + + # edge case - if a user tries to load a sentence_transformer model but does not pass a model name + if from_sentence_transformer and not embedding_model_name: + raise LLMWareException(message=f"Query - init - to use sentence_transformers, please " + f"provide the model name directly to load") + + # load default configs + # embedding initialization parameters + self.query_embedding = None + self.embedding_model = None + self.embedding_db = None + self.embeddings = None + self.model_api_key = model_api_key + + if self.library: + self.embeddings = EmbeddingHandler(self.library) + + self.semantic_distance_threshold = 1000 # basic shut off at such a high level + + # keys that will be included in query results + + # full list + self.query_result_standard_keys = ["_id", "text", "doc_ID", "block_ID","page_num","content_type", + "author_or_speaker", "special_field1", "file_source","added_to_collection", + "table", "coords_x", "coords_y", "coords_cx", "coords_cy", "external_files", + "score", "similarity", "distance", "matches"] + + # short_list + self.query_result_short_keys = ["text", "file_source", "page_num", "score", "distance","matches"] + + # minimum_list + self.query_result_min_required_keys = ["text", "file_source", "page_num"] + + # default - set at 'full list' + self.query_result_return_keys = self.query_result_standard_keys + + # default is semantic if embedding in place + embedding_record = self.library.get_embedding_status() + + matched_lib_model = False + + if embedding_model_name: + for emb in embedding_record: + + logger.info(f"update: Query - embedding record lookup - {embedding_model_name} - {emb}") + + if emb["embedding_model"] == embedding_model_name: + + # if no vector_db name passed, then select based only on embedding_model + if not vector_db: + if emb["embedding_status"] == "yes": + self.embedding_db = emb["embedding_db"] + self.search_mode = "semantic" + matched_lib_model = True + break + else: + # confirm match of pair - embedding_model + vector_db + if emb["embedding_db"] == vector_db: + if emb["embedding_status"] == "yes": + self.embedding_db = emb["embedding_db"] + self.search_mode = "semantic" + matched_lib_model = True + break + + else: + if len(embedding_record) > 0: + + if not vector_db: + last_emb_record = embedding_record[-1] + if last_emb_record["embedding_status"] == "yes": + self.embedding_db = last_emb_record["embedding_db"] + self.search_mode = "semantic" + self.embedding_model_name = last_emb_record["embedding_model"] + matched_lib_model = True + else: + # look for match to passed vector_db and take most recent embedding + embedding_record.reverse() + for embs in embedding_record: + if embs["embedding_db"] == vector_db: + if embs["embedding_status"] == "yes": + self.embedding_db = vector_db + self.search_mode = "semantic" + self.embedding_model_name = embs["embedding_model"] + matched_lib_model = True + break + + if matched_lib_model: + + logger.info(f"update: Query - found matches in embedding record - " + f"{self.embedding_db} - {self.embedding_model_name}") + + if not self.embedding_model: + self.load_embedding_model() + + else: + self.search_mode = "text" + + # passed for accessing api_based vector db + self.vector_db_api_key = vector_db_api_key + + # if query_id passed, then load that state + if query_id: + self.query_id = query_id + self.load_query_state(query_id) + else: + self.query_id = QueryState().issue_new_query_id() + + self.result_text_chunk_size = self.library.block_size_target_characters + + # state variables + self.results = [] + self.query_history = [] + self.doc_id_list = [] + self.doc_fn_list = [] + + self.save_history = save_history + + if query_mode: + self.search_mode = query_mode + + # confirm that 'query_history' path exists + query_history_path = LLMWareConfig().get_query_path() + if not os.path.exists(query_history_path): + os.mkdir(query_history_path) + os.chmod(query_history_path, 0o777) + + def load_embedding_model(self): + + """ Loads the embedding model pulled from the embedding_record of the library. """ + + # skip if already instantiated self.embedding_model + + if not self.embedding_model: + + if self.user_passed_model: + + if self.from_hf: + self.embedding_model = ModelCatalog().load_hf_embedding_model(self.user_passed_model, + self.user_passed_tokenizer) + if self.from_sentence_transformer: + self.embedding_model = ModelCatalog().load_sentence_transformer_model(self.user_passed_model, + self.embedding_model_name) + + else: + if ModelCatalog().lookup_model_card(self.embedding_model_name): + self.embedding_model = ModelCatalog().load_model(selected_model=self.embedding_model_name, + api_key=self.model_api_key) + else: + logger.info(f"update: Query - selected embedding model could not be found - " + f"{self.embedding_model_name}") + + return self + + def get_output_keys(self): + + """ Returns list of keys that will be provided in each query_result. """ + + return self.query_result_return_keys + + def set_output_keys(self, result_key_list): + + """ Sets the list of keys that will be returned in each query_result. """ + + # set the output keys + validated_list = [] + for key in result_key_list: + if key in self.library.default_keys: + validated_list.append(key) + + # minimum required list + for rk in self.query_result_min_required_keys: + if rk not in validated_list: + validated_list.append(rk) + logger.info(f"warning: Query - adding required keys useful in downstream processing - {rk}") + + # setting updated query_return_keys that is used in packaging query results + self.query_result_return_keys = validated_list + + return validated_list + + def start_query_session(self, query_id=None): + + """ Initiates a query session and will capture potentially multiple related queries in single state. """ + + if query_id: + self.query_id = query_id + + if self.query_id: + QueryState(self).load_state(self.query_id) + else: + query_id = QueryState(self).initiate_new_state_session() + + return query_id + + def register_query (self, retrieval_dict): + + """ Registers a query to the query state. """ + + # qr_dict = ["query", "results", "doc_ID", "file_source"] + + # add query results as new "column" in query state + self.results += retrieval_dict["results"] + + if retrieval_dict["query"] not in self.query_history: + self.query_history.append(retrieval_dict["query"]) + + for doc_id in retrieval_dict["doc_ID"]: + if doc_id not in self.doc_id_list: + self.doc_id_list.append(doc_id) + + for doc_fn in retrieval_dict["file_source"]: + if doc_fn not in self.doc_fn_list: + self.doc_fn_list.append(doc_fn) + + # QueryState(self).save_state(self.query_id) + + return self + + def load_query_state(self, query_id): + + """ Loads a query state of a previous query by query_id """ + + state = QueryState(self).load_state(query_id) + return self + + def save_query_state(self): + + """ Saves the current query state. """ + + QueryState(self).save_state() + return self + + def clear_query_state(self): + + """ Resets the query state. """ + + # need to reset state variables + QueryState(self).initiate_new_state_session() + return self + + def dump_current_query_state(self): + + """ Dumps the current query_state to a query_state_dict. """ + + query_state_dict = {"query_id": self.query_id, + "query_history": self.query_history, + "results": self.results, + "doc_ID": self.doc_id_list, + "file_source": self.doc_fn_list + } + + return query_state_dict + + def query(self, query, query_type="text", result_count=20, results_only=True): + + """ Main method for executing a basic query - expects query as input, and optional parameters for + query_type, result_count and whether results_only. Output is a set of query results, which is a list of + dictionaries, with each dictionary representing a single matching retrieval from the collection. """ + + output_result = {"results": [], "doc_ID": [], "file_source": []} + + if query_type not in ["text", "semantic"]: + logger.error("error: Query().query expects a query type of either 'text' or 'semantic'") + return output_result + + if query_type == "text": + output_result = self.text_query(query,result_count=result_count,results_only=results_only) + + if query_type == "semantic": + + # check that embedding model is available, and if not, flip to text search + + if not self.embedding_model: + self.load_embedding_model() + + if self.search_mode == "text" or not self.embedding_model: + output_result = self.text_query(query, result_count=result_count,results_only=results_only) + else: + output_result = self.semantic_query(query, result_count=result_count,results_only=results_only) + + return output_result + + def text_query (self, query, exact_mode=False, result_count=20, exhaust_full_cursor=False, results_only=True): + + """ Execute a basic text query. """ + + # prepare query if exact match required + if exact_mode: + query = self.exact_query_prep(query) + + # query the text collection + cursor = CollectionRetrieval(self.library_name,account_name=self.account_name).basic_query(query) + + # package results, with correct sample counts and output keys requested + results_dict = self._cursor_to_qr(query, cursor,result_count=result_count,exhaust_full_cursor= + exhaust_full_cursor) + + if results_only: + return results_dict["results"] + + return results_dict + + def text_query_with_document_filter(self, query, doc_filter, result_count=20, exhaust_full_cursor=False, + results_only=True, exact_mode=False): + + """ Execute a text query with a document filter applied. """ + + # prepare query if exact match required + if exact_mode: + query = self.exact_query_prep(query) + + key = None + value_range = [] + + if "doc_ID" in doc_filter: + key = "doc_ID" + value_range = doc_filter["doc_ID"] + + elif "file_source" in doc_filter: + key = "file_source" + value_range = doc_filter["file_source"] + + else: + logger.warning("warning: Query - expected to receive document filter with keys of 'doc_ID' or " + "'file_source' - as a safe fall-back - will run the requested query without a filter.") + + if key: + cursor = CollectionRetrieval(self.library_name, account_name=self.account_name). \ + text_search_with_key_value_range(query, key, value_range) + else: + # as fallback, if no key found, then run query without filter + cursor = CollectionRetrieval(self.library_name, account_name=self.account_name).basic_query(query) + + result_dict = self._cursor_to_qr(query, cursor, result_count=result_count, + exhaust_full_cursor=exhaust_full_cursor) + + if results_only: + return result_dict["results"] + + return result_dict + + def text_query_by_content_type (self, query, content_type,results_only=True): + + """ Execute a text query with additional constraint of content type, e.g., 'image', 'table', or 'text' """ + + filter_dict = {"content_type": content_type} + retrieval_dict = self.text_query_with_custom_filter(query,filter_dict,results_only=True) + return retrieval_dict + + def image_query(self, query, results_only=True): + + """ Execute a query with content_type == 'image'. """ + + filter_dict = {"content_type": "image"} + retrieval_dict = self.text_query_with_custom_filter(query, filter_dict,results_only=True) + return retrieval_dict + + def table_query(self, query, export_tables_to_csv=False, results_only=True): + + """ Execute a query with content_type == 'table'. """ + + filter_dict = {"content_type": "table"} + retrieval_dict = self.text_query_with_custom_filter(query, filter_dict,results_only=True) + + # output and write tables to csv files + if export_tables_to_csv: + for i, entry in enumerate(retrieval_dict["results"]): + f = self.export_one_table_to_csv(entry,output_fp=LLMWareConfig.get_query_path(), + output_fn="table_{}.csv".format(i)) + + logger.warning(f"update: csv created - {LLMWareConfig.get_query_path()}- {f}") + + return retrieval_dict + + def text_search_by_page (self, query, page_num=1, results_only=True): + + """ Execute a text search with page number constraint provided as page_num parameter. """ + + key = "master_index" # parsing uses "master_index" across multiple input sources, interpret as "page_num" + + if not isinstance(page_num, list): + page_num = [page_num] + + cursor_results = CollectionRetrieval(self.library_name, account_name=self.account_name).\ + text_search_with_key_value_range(query, key, page_num) + + retrieval_dict = self._cursor_to_qr(query, cursor_results) + + if results_only: + return retrieval_dict["results"] + + return retrieval_dict + + def text_query_by_author_or_speaker(self, query, author_or_speaker, results_only=True): + + """ Execute a text query with specific author_or_speaker constraint. """ + + filter_dict = {"author_or_speaker": author_or_speaker} + retrieval_dict = self.text_query_with_custom_filter(query,filter_dict,results_only=results_only) + return retrieval_dict + + def text_query_with_custom_filter (self, query, filter_dict, result_count=20, + exhaust_full_cursor=False, results_only=True): + + """ Execute a text query with additional custom filter dictionary. """ + + # filter_dict is a dict with indefinite number of key:value pairs - each key will be interpreted + # as "$and" in the query, requiring a match against all of the key:values in the filter_dict + + # validate filter dict + validated_filter_dict = {} + for key, values in filter_dict.items(): + for valid_keys in self.library.default_keys: + if key in valid_keys: + validated_filter_dict.update({key:values}) + + if validated_filter_dict: + cursor = CollectionRetrieval(self.library_name, account_name=self.account_name).\ + text_search_with_key_value_dict_filter(query,validated_filter_dict) + + else: + logger.error("error: Query text_query_with_custom_filter - keys in filter_dict are not" + "recognized as part of the library.collection default_keys list.") + + return -1 + + result_dict = self._cursor_to_qr_with_secondary_filter(query, cursor,filter_dict, + result_count=result_count, + exhaust_full_cursor=exhaust_full_cursor) + + if results_only: + return result_dict["results"] + + return result_dict + + def _cursor_to_qr_with_secondary_filter(self, query, cursor_results, filter_dict, + result_count=20, exhaust_full_cursor=False): + + """ Internal helper method to package query results from cursor. """ + + qr = [] + counter = 0 + doc_id_list = [] + doc_fn_list = [] + + for raw_qr in cursor_results: + + # update to locate match and add to result + matches_found = self.locate_query_match(query, raw_qr["text"]) + raw_qr.update({"matches": matches_found}) + raw_qr.update({"page_num": raw_qr["master_index"]}) + + raw_qr.update({"_id": str(raw_qr["_id"])}) + + if "score" not in raw_qr: + raw_qr.update({"score": 0.0}) + + if "similarity" not in raw_qr: + raw_qr.update({"similarity": 0.0}) + + if "distance" not in raw_qr: + raw_qr.update({"distance": 0.0}) + + # apply secondary filter dict + match = -1 + for key, value in filter_dict.items(): + if key in raw_qr: + # support case in which filter_dict is a list, e.g., doc_id = {5,9,13} + if raw_qr[key] == value or (isinstance(value,list) and raw_qr[key] in value): + match = 1 + else: + match = -1 + break + + if match == 1: + + # output target keys + output_dict = {} + output_dict.update({"query": query}) + + for key in self.query_result_return_keys: + if key not in raw_qr: + logger.warning(f"warning: Query() - selected output key not found in result - {key}") + else: + output_dict.update({key: raw_qr[key]}) + + output_dict.update({"account_name": self.account_name}) + output_dict.update({"library_name": self.library_name}) + + qr.append(output_dict) + + if raw_qr["doc_ID"] not in doc_id_list: + doc_id_list.append(raw_qr["doc_ID"]) + doc_fn_list.append(raw_qr["file_source"]) + + counter += 1 + + # will exhaust full cursor if .exhaust_full_cursor = True + if counter >= result_count and not exhaust_full_cursor: + break + + qr_dict = {"query": query, "results": qr, "doc_ID": doc_id_list, "file_source": doc_fn_list} + + if self.save_history: + self.register_query(qr_dict) + + return qr_dict + + def _cursor_to_qr (self, query, cursor_results, result_count=20, exhaust_full_cursor=False): + + """ Internal helper method to package query results from cursor. """ + + qr = [] + counter = 0 + doc_id_list = [] + doc_fn_list = [] + + for raw_qr in cursor_results: + + # update to locate match and add to result + matches_found = self.locate_query_match(query, raw_qr["text"]) + raw_qr.update({"matches": matches_found}) + raw_qr.update({"page_num": raw_qr["master_index"]}) + + raw_qr.update({"_id": str(raw_qr["_id"])}) + + if "score" not in raw_qr: + raw_qr.update({"score": 0.0}) + + if "similarity" not in raw_qr: + raw_qr.update({"similarity": 0.0}) + + if "distance" not in raw_qr: + raw_qr.update({"distance": 0.0}) + + # output target keys + output_dict = {} + output_dict.update({"query": query}) + + for key in self.query_result_return_keys: + if key not in raw_qr: + logger.warning(f"warning: Query() - selected output key not found in result - {key}") + else: + output_dict.update({key: raw_qr[key]}) + + output_dict.update({"account_name": self.account_name}) + output_dict.update({"library_name": self.library_name}) + + qr.append(output_dict) + + if raw_qr["doc_ID"] not in doc_id_list: + doc_id_list.append(raw_qr["doc_ID"]) + doc_fn_list.append(raw_qr["file_source"]) + + counter += 1 + + # will exhaust full cursor if .exhaust_full_cursor = True + if counter >= result_count and not exhaust_full_cursor: + break + + qr_dict = {"query": query,"results": qr, "doc_ID": doc_id_list, "file_source": doc_fn_list} + + if self.save_history: + self.register_query(qr_dict) + + return qr_dict + + def semantic_query(self, query, result_count=20, embedding_distance_threshold=None, custom_filter=None, results_only=True): + + """ Main method to execute a semantic query - only required parameter is the query. """ + + if not embedding_distance_threshold: + embedding_distance_threshold = self.semantic_distance_threshold + + self.load_embedding_model() + + # confirm that embedding model exists, or catch and raise error + if self.embedding_model: + self.query_embedding = self.embedding_model.embedding(query) + else: + raise ModelNotFoundException(self.library_name) + + if self.embedding_db and self.embedding_model: + + semantic_block_results = self.embeddings.search_index(self.query_embedding, + embedding_db=self.embedding_db, + model=self.embedding_model, + sample_count=result_count) + + else: + logger.error(f"error: Query - embedding record does not indicate embedding db - " + f"{self.embedding_db} and/or embedding model - {self.embedding_model}") + + raise LLMWareException(message=f"Query - semantic query - selected " + f"embedding database is not supported - {self.embedding_db}") + + qr_raw = [] + + # Collecting semantic results + for i, blocks in enumerate(semantic_block_results): + if blocks[1] < embedding_distance_threshold: + block_data = blocks[0] + block_data["distance"] = blocks[1] + block_data["semantic"] = "semantic" + block_data["score"] = 0.0 + qr_raw.append(block_data) + + # Applying custom filter if provided + if custom_filter: + qr_raw = self.apply_custom_filter(qr_raw, custom_filter) + + # Processing results + results_dict = self._cursor_to_qr(query, qr_raw, result_count=result_count) + + return results_dict["results"] if results_only else results_dict + + def apply_custom_filter(self, results, custom_filter): + + """ Apply custom filter to a set of results. """ + + filtered_results = [] + for result in results: + if all(result.get(key) == value for key, value in custom_filter.items()): + filtered_results.append(result) + return filtered_results + + def semantic_query_with_document_filter(self, query, filter_dict, embedding_distance_threshold=None, + result_count=100, results_only=True): + + """ Execute semantic query with secondary constraint of document filter. """ + + # checks for filter to offer option to do semantic query in specific doc, page or content range + if not embedding_distance_threshold: + embedding_distance_threshold = self.semantic_distance_threshold + + # note: by default, retrieves a much larger set of results to try to account for filter + + th = self.semantic_distance_threshold + + # confirm that embedding model exists, or catch and raise error + if self.embedding_model: + self.query_embedding = self.embedding_model.embedding(query) + else: + raise ModelNotFoundException(self.library_name) + + if self.embedding_db and self.embedding_model: + semantic_block_results = self.embeddings.search_index(self.query_embedding, + embedding_db=self.embedding_db, + model=self.embedding_model, + sample_count=result_count) + + else: + logger.error(f"error: Query - embedding record does not indicate embedding db- {self.embedding_db} " + f"and/or an embedding_model - {self.embedding_model}") + + raise LLMWareException(message=f"Query - semantic query with document filter - selected " + f"embedding database is not supported - {self.embedding_db}") + + qr_raw = [] + + # may need to conform the output structure of semantic_block_results + for i, blocks in enumerate(semantic_block_results): + # assume that each block has at least two components: [0] core mongo block, and [1] distance metric + if blocks[1] < embedding_distance_threshold: + + blocks[0].update({"distance": blocks[1]}) + blocks[0].update({"semantic": "semantic"}) + blocks[0].update({"score": 0.0}) + + qr_raw.append(blocks[0]) + + result_output = self._cursor_to_qr_with_secondary_filter(query,qr_raw,filter_dict,result_count=result_count) + + if results_only: + return result_output["results"] + + return result_output + + def similar_blocks_embedding(self, block, result_count=20, embedding_distance_threshold=10, results_only=True): + + """ Application of semantic embedding space - takes a block of text as input and finds most similar comparable + text chunks. If you are comfortable with the normalization of the embedding vector space, then set a + specific embedding_distance_threshold, e.g., embedding_distance_threshold=1.1 to limit the results to + text blocks within a distance of 1.1 in the embedding space.""" + + # will use embedding to find similar blocks from a given block + # confirm that embedding model exists, or catch and raise error + if self.embedding_model: + self.query_embedding = self.embedding_model.embedding(block["text"]) + else: + raise ModelNotFoundException(self.library_name) + + if self.embedding_model and self.embedding_db: + semantic_block_results = self.embeddings.search_index(self.query_embedding, + embedding_db=self.embedding_db, + model=self.embedding_model, + sample_count=result_count) + + else: + logger.error(f"error: Query - embedding record does not indicate embedding db- " + f"{self.embedding_db} and/or embedding model - {self.embedding_model}") + + raise LLMWareException(message=f"Query - similar blocks embedding - selected " + f"embedding database is not supported - {self.embedding_db}") + + qr_raw = [] + + # may need to conform the output structure of semantic_block_results + for i, blocks in enumerate(semantic_block_results): + # assume that each block has at least two components: [0] core mongo block, and [1] distance metric + if blocks[1] < embedding_distance_threshold: + + blocks[0].update({"distance": blocks[1]}) + blocks[0].update({"semantic": "semantic"}) + blocks[0].update({"score": 0.0}) + + qr_raw.append(blocks[0]) + + # pick up with boilerplate + results_dict = self._cursor_to_qr("", qr_raw, result_count=result_count) + + if results_only: + return results_dict["results"] + + return results_dict + + def dual_pass_query(self, query, result_count=20, primary="text", + safety_check=True, custom_filter=None, results_only=True): + + """ Executes a combination of text and semantic queries and attempts to interweave and re-rank based on + correspondence between the two query attempts. """ + + # safety check + if safety_check and result_count > 100: + logger.info("warning: Query().dual_pass_query runs a comparison of output rankings using semantic " + "and text. This particular implementation is not optimized for sample lists longer " + "than ~100 X 100. To remove this warning, there are two options - (1) set the " + "safety_check to False in the method declaration, or (2) keep sample count below 100.") + + result_count = 100 + + # following keys are required for dual pass query to work, add them if user has omitted them + keys_to_check = ['_id', 'doc_ID'] + for key in keys_to_check: + if key not in self.query_result_return_keys: + self.query_result_return_keys.append(key) + + # run dual pass - text + semantic + # Choose appropriate text query method based on custom_filter + if custom_filter: + retrieval_dict_text = self.text_query_with_custom_filter(query, custom_filter, + result_count=result_count, results_only=True) + else: + retrieval_dict_text = self.text_query(query, result_count=result_count, results_only=True) + + # Semantic query with custom filter + retrieval_dict_semantic = self.semantic_query(query, result_count=result_count, + custom_filter=custom_filter, results_only=True) + + if primary == "text": + first_list = retrieval_dict_text + second_list = retrieval_dict_semantic + else: + first_list = retrieval_dict_semantic + second_list = retrieval_dict_text + + confirming_list = [] + primary_only = [] + secondary_only = [] + matched_second_list = [] + + # this is the time intensive "n-squared" loop - probably OK up to 100+ + for i, entry in enumerate(first_list): + match = -1 + for j, entry2 in enumerate(second_list): + if entry["_id"] == entry2["_id"]: + entry["match_status"] = "matched" # Tagging as matched + confirming_list.append(entry) + match = 1 + matched_second_list.append(entry2["_id"]) + break + if match == -1: + entry["match_status"] = "primary_only" # Tagging as primary only + primary_only.append(entry) + + for k, entry2 in enumerate(second_list): + if entry2["_id"] not in matched_second_list: + entry2["match_status"] = "secondary_only" # Tagging as secondary only + secondary_only.append(entry2) + + # assemble merged top results + merged_results = [] + merged_results += confirming_list + + select_primary = min(len(primary_only), 5) + select_secondary = min(len(secondary_only), 5) + + merged_results += primary_only[0:select_primary] + merged_results += secondary_only[0:select_secondary] + + doc_id_list = [] + doc_fn_list = [] + + for qr in merged_results: + if qr["doc_ID"] not in doc_id_list: + doc_id_list.append(qr["doc_ID"]) + if qr["file_source"] not in doc_fn_list: + doc_fn_list.append(qr["file_source"]) + + retrieval_dict = {"results": merged_results, + "text_results": retrieval_dict_text, + "semantic_results": retrieval_dict_semantic, + "doc_ID": doc_id_list, + "file_source": doc_fn_list} + + if results_only: + return merged_results + + return retrieval_dict + + def augment_qr (self, query_result, query_topic, augment_query="semantic"): + + """ Augments the set of query results using alternative retrieval strategy. """ + + if augment_query == "semantic": + qr_aug = self.semantic_query(query_topic,result_count=20, results_only=True) + else: + qr_aug = self.text_query(query_topic,result_count=20, results_only=True) + + # consolidate the qr lists + updated_qr = [] + for qr in query_result: + updated_qr.append(qr) # start with original qr list + + # add up to 10 entries from semantic list + semantic_return_max = 10 + + for j, sem_entries in enumerate(qr_aug): + if sem_entries not in updated_qr: + updated_qr.append(sem_entries) + if j > semantic_return_max: + break + + return updated_qr + + def apply_semantic_ranking(self, qr, issue_semantic): + + """ Apply ranking of query results by semantic query ranking. """ + + # designed to take a set of query results, and re-rank the order of results by their semantic distance + # --note: possible to use a different query term for issue_semantic than the original query result + + # heuristic - look for result targets of at least 20, but up to the exact len of the qr + result_target = max(len(qr),20) + + semantic_qr = self.semantic_query(issue_semantic,result_count=result_target) + + reranked_qr = [] + for i, s in enumerate(semantic_qr): + + for q in qr: + if s["_id"] == q["_id"]: + reranked_qr.append(q) + break + + for q in qr: + if q not in reranked_qr: + reranked_qr.append(q) + + return reranked_qr + + def document_filter (self, filter_topic, query_mode="text", result_count=30, + exact_mode = False, exhaust_full_cursor=True): + + """ Takes a filter_topic as input, along with query_mode, and generates a document filter as output. """ + + result_dict = None + + if query_mode not in ["text", "semantic", "hybrid"]: + + logger.error(f"error: Query document_filter supports query types - 'text', " + f"'semantic', and 'hybrid' - type selected not recognized - {query_mode}") + + return result_dict + + if query_mode == "text": + result_dict = self.text_query(filter_topic,exact_mode=exact_mode,result_count=result_count, + exhaust_full_cursor=exhaust_full_cursor,results_only=False) + + if query_mode == "semantic": + result_dict = self.semantic_query(filter_topic,result_count=result_count, results_only=False) + + if query_mode == "hybrid": + result_dict = self.dual_pass_query(filter_topic) + + if not result_dict: + + logger.error(f"error: Query file_selector_only could not find a result - unexpected error - " + f"{filter_topic}") + + return result_dict + + doc_filter_output = {"doc_ID": result_dict["doc_ID"], "file_source": result_dict["file_source"]} + + return doc_filter_output + + def page_lookup(self, page_list=None, doc_id_list=None, text_only=False): + + """ Look up by specific pages, e.g, useful to retrieve the entire first page of a template document. """ + + if not doc_id_list: + doc_id_list = self.list_doc_id() + else: + if isinstance(doc_id_list,dict): + if "doc_ID" in doc_id_list: + doc_id_list = doc_id_list["doc_ID"] + else: + logger.warning("warning: could not recognize doc id list requested. by default, " + "will set to all documents in the library collection.") + + doc_id_list = self.list_doc_id() + + if not page_list: + logger.warning("warning: page lookup requested, but no value range identified. by default, will set " + "to retrieve the first page only.") + page_list = [1] + + if text_only: + page_dict = {"doc_ID": {"$in":doc_id_list}, "master_index": {"$in": page_list}, "content_type":"text"} + else: + page_dict = {"doc_ID": {"$in":doc_id_list}, "master_index": {"$in": page_list}} + + cursor_results = CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key_dict(page_dict) + + output = [] + + for x in cursor_results: + + x.update({"matches": []}) + x.update({"page_num": x["master_index"]}) + + output.append(x) + + return output + + def get_whole_library(self, selected_keys=None): + + """ Gets the whole library - and will return as a list in-memory. """ + + match_results_cursor = CollectionRetrieval(self.library_name, + account_name=self.account_name).get_whole_collection() + + match_results = match_results_cursor.pull_all() + + qr = [] + + # option to retrieve only user selected keys + if not selected_keys: + selected_keys = self.library.default_keys + + for i, block in enumerate(match_results): + + new_row = {} + new_row.update({"_id": str(block["_id"])}) + new_row.update({"matches": []}) + new_row.update({"page_num": block["master_index"]}) + new_row.update({"score": 0.0}) + new_row.update({"similarity": 0.0}) + new_row.update({"distance": 0.0}) + + for keys in selected_keys: + if keys in block: + if keys not in new_row: + new_row.update({keys:block[keys]}) + + qr.append(new_row) + + return qr + + def export_all_tables(self, query="", output_fp=None): + + """ Exports all tables, with query option to limit the list from a library. """ + + table_csv_files_created = [] + + if not output_fp: + output_fp = self.library.misc_path + + if not query: + + match_results = CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key("content_type","table") + + else: + kv_dict = {"content_type": "table"} + match_results = CollectionRetrieval(self.library_name, account_name=self.account_name).\ + text_search_with_key_value_dict_filter(query,kv_dict) + + counter = 0 + + for i, entries in enumerate(match_results): + + table = entries["table"] + + output = [] + + table_raw = table + rows = table_raw.split("") + cols_tracker = [] + coords_master = [] + + for row in rows: + + new_row = [] + if row.strip().endswith(""): + row = row.strip()[:-5] + + cells = row.lstrip().split("") + cols_count = 0 + coords = [] + + for c in cells: + + if c.strip().endswith(""): + c = c.strip()[:-5] + + clean_cell = "" + bracket_on = 0 + + fields = c.split("<") + + if fields[0]: + index = fields[1].rstrip()[0:-1] + + main_entry = fields[2].split(">") + value = main_entry[-1] + + co = main_entry[0].split(" ") + + if len(co) > 2: + x = co[1] + y = co[2] + + coords.append((int(x), int(y))) + + for c1 in c: + if bracket_on == 0 and c1 not in ("<", ">"): + clean_cell += c1 + if c1 == "<": + bracket_on = 1 + if c1 == ">": + bracket_on = 0 + + if c: + c_strip = c.split(">")[-1] + new_row.append(c_strip.strip()) + cols_count += 1 + + coords_master.append(coords) + cols_tracker.append(cols_count) + output.append(new_row) + + new_file = "table_{}.csv".format(counter) + + counter += 1 + f = Utilities().file_save(output, output_fp, new_file) + output = [] + + table_csv_files_created.append(new_file) + + output_dict = {"library": self.library_name, "query": query, "tables_created": counter, + "file_names": table_csv_files_created, "output_fp": output_fp} + + return output_dict + + def export_one_table_to_csv(self, query_result, output_fp=None, output_fn=None): + + """ Exports a single table query result into a csv file. """ + + table = query_result["table"] + + output = [] + + table_raw = table + rows = table_raw.split("") + cols_tracker = [] + coords_master = [] + + for row in rows: + + new_row = [] + if row.strip().endswith(""): + row = row.strip()[:-5] + + cells = row.lstrip().split("") + cols_count = 0 + coords = [] + + for c in cells: + + if c.strip().endswith(""): + c = c.strip()[:-5] + + clean_cell = "" + bracket_on = 0 + + fields = c.split("<") + + if fields[0]: + index = fields[1].rstrip()[0:-1] + main_entry = fields[2].split(">") + value = main_entry[-1] + co = main_entry[0].split(" ") + if len(co) > 2: + x = co[1] + y = co[2] + coords.append((int(x), int(y))) + + for c1 in c: + if bracket_on == 0 and c1 not in ("<", ">"): + clean_cell += c1 + if c1 == "<": + bracket_on = 1 + if c1 == ">": + bracket_on = 0 + + if c: + c_strip = c.split(">")[-1] + new_row.append(c_strip.strip()) + cols_count += 1 + coords_master.append(coords) + cols_tracker.append(cols_count) + output.append(new_row) + + if not output_fn: + new_file = "table_0.csv" + else: + new_file = output_fn + + f = Utilities().file_save(output, output_fp, new_file) + + return new_file + + def list_doc_id(self): + + """ Utility function - returns list of all doc_ids in the library. """ + + doc_id_list = CollectionRetrieval(self.library_name, account_name=self.account_name).get_distinct_list("doc_ID") + + return doc_id_list + + def list_doc_fn(self): + + """ Utility function - returns list of all document names in the library. """ + + doc_fn_raw_list = CollectionRetrieval(self.library_name, + account_name=self.account_name).get_distinct_list("file_source") + + doc_fn_out = [] + for i, file in enumerate(doc_fn_raw_list): + doc_fn_out.append(file.split(os.sep)[-1]) + return doc_fn_out + + def aggregate_text(self, qr_list): + + """ Utility method that take a list of query result dictionaries as input (with all of the associated metadata + attributes) and repackages into two useful outputs: + + -- text_agg, which is the aggregated text across all of the query results in a single unbroken string, and + -- meta_agg, which is a list of dictionaries with all of the 'start/stop' information in the text + string, which can be used to map back a snippet of text back to its original block entry in the DB. + """ + + text_agg = "" + meta_agg = [] + + for i, entry in enumerate(qr_list): + t = entry["text"] + meta = {"start_char": len(text_agg), "stop_char": len(text_agg) + len(t), "block_id": entry["block_ID"], + "doc_ID": entry["doc_ID"], "page_num": entry["page_num"]} + meta_agg.append(meta) + text_agg += t + + return text_agg, meta_agg + + def document_lookup(self, doc_id="", file_source="", include_images=False): + """ + Takes as an input either a doc_id or file_source (e.g., filename) that is in a Library, and + returns all of the text and table blocks in the document. Images can be optionally included. + + Parameters: + doc_id (str): Document ID. + file_source (str): Source file name. + include_images (bool): Whether to include images in the result. Defaults to False. + + Returns: + list: Filtered list of document blocks. + """ + + if doc_id: + kv_dict = {"doc_ID": doc_id} + elif file_source: + kv_dict = {"file_source": file_source} + else: + raise RuntimeError( + "Query document_lookup method requires as input either a document ID or " + "the name of a file already parsed in the library" + ) + + output = CollectionRetrieval(self.library_name, account_name=self.account_name).filter_by_key_dict(kv_dict) + + if len(output) == 0: + logger.warning(f"update: Query - document_lookup - nothing found - {doc_id} - {file_source}") + return [] + + output_final = [] + + for entries in output: + # Filter out images if include_images is False + if include_images or entries["content_type"] != "image": + entries.update({"matches": []}) + entries.update({"page_num": entries["master_index"]}) + output_final.append(entries) + + output_final = sorted(output_final, key=lambda x: x["block_ID"], reverse=False) + + return output_final + + def block_lookup(self, block_id, doc_id): + + """ Look up by a specific pair of doc_id and block_id in a library. """ + + result = None + + kv_dict = {"doc_ID": doc_id, "block_ID": block_id} + + output = CollectionRetrieval(self.library_name, account_name=self.account_name).filter_by_key_dict(kv_dict) + + if len(output) == 0: + logger.info(f"update: Query - Library - block_lookup - block not found: {block_id}") + result = None + + return result + + if len(output) > 1: + result = output[0] + + if len(output) == 1: + result = output[0] + + # if arrived this point, then positive result has been identified + result.update({"matches": []}) + result.update({"page_num": result["master_index"]}) + + + return result + + def get_header_text_from_collection(self, text_field="header_text"): + + """ Pulls the header_text from the collection, captured during parsing, useful to identify headlines. """ + + ds_folder = self.library.nlp_path + + results = CollectionRetrieval(self.library_name, account_name=self.account_name).get_whole_collection() + + f = open(ds_folder + "header_text.txt", "w", encoding='utf-8') + counter = 0 + for elements in results: + text_sample = elements[text_field] + if text_sample: + f.write(text_sample) + f.write("\n") + f.write(elements["text"]) + f.write("\n") + counter += 1 + + f.close() + results.close() + return counter + + def get_core_text_from_collection(self, text_field="text"): + + """ Returns all core text from a collection. """ + + ds_folder = self.library.nlp_path + + results = CollectionRetrieval(self.library_name, account_name=self.account_name).get_whole_collection() + + f = open(os.path.join(ds_folder,"core_text.txt"), "w", encoding='utf-8') + counter = 0 + for elements in results: + text_sample = elements[text_field] + if text_sample: + f.write(text_sample) + f.write("\n") + counter += 1 + + f.close() + results.close() + return counter + + def get_user_tags(self): + + """ Returns user tags, if any, identified - note: this is experimental method and may change.""" + + # look for all non-empty user_tags + output = CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key_ne_value("user_tags", "") + + counter = 0 + user_tags_out = [] + for elements in output: + counter += 1 + user_tags_out.append((elements["block_ID"], elements["user_tags"])) + + return user_tags_out + + def filter_by_time_stamp (self, qr, first_date="", last_date=""): + + """ Filters results by condition of time range. """ + + # apply filter dict to the qr results found + time_str = "%Y-%m-%d" + if first_date: + first_date = datetime.strptime(first_date,time_str) + + if last_date: + last_date = datetime.strptime(last_date, time_str) + + filtered_qr = [] + + for i, entry in enumerate(qr): + + if entry["added_to_collection"]: + + try: + # First try Linux format + time_str = "%a %b %d %H:%M:%S %Y" + doc_date = datetime.strptime(entry["added_to_collection"], time_str) + + except ValueError: + # If it fails, try Windows format + time_str = "%m/%d/%y %H:%M:%S" + doc_date = datetime.strptime(entry["added_to_collection"], time_str) + + time_accept = self._time_window_filter(first_date,last_date,doc_date) + + if time_accept: + filtered_qr.append(entry) + + return filtered_qr + + def _time_window_filter(self, start_time,end_time, test_time, time_str="%a %b %d %H:%M:%S %Y"): + + """ Internal helper function to evaluate and compare time stamps. """ + + if start_time and end_time: + if start_time <= test_time <= end_time: + return True + + if start_time and not end_time: + if start_time <= test_time: + return True + + if not start_time and end_time: + if test_time <= end_time: + return True + + return False + + def locate_query_match (self, query, core_text): + + """ Utility function to locate the character-level match of a query inside a core_text. """ + + matches_found = [] + + # edge case - but return empty match if query is null + if not query: + return matches_found + + b = CorpTokenizer(one_letter_removal=False, remove_stop_words=False, remove_punctuation=False, + remove_numbers=False) + + query_tokens = b.tokenize(query) + + for x in range(0, len(core_text)): + match = 0 + for key_term in query_tokens: + if len(key_term) == 0: + continue + + if key_term.startswith('"'): + key_term = key_term[1:-1] + + if core_text[x].lower() == key_term[0].lower(): + match += 1 + if (x + len(key_term)) <= len(core_text): + for y in range(1, len(key_term)): + if key_term[y].lower() == core_text[x + y].lower(): + match += 1 + else: + match = -1 + break + + if match == len(key_term): + new_entry = [x, key_term] + matches_found.append(new_entry) + + return matches_found + + def exact_query_prep(self, query): + + """ Prepares an exact query prep. """ + + if query.startswith('"') and query.endswith('"'): + prepared_query = '\"' + query[1:-1] + '\"' + + else: + # even if user did not wrap in quotes, treat as exact search + prepared_query = '\"' + query + '\"' + + return prepared_query + + def bibliography_builder_from_qr(self, query_results): + + """ Builds a bibliography from a query result. """ + + bibliography = [] + doc_id_reviewed = [] + doc_fn_reviewed = [] + + # first - assemble the list of docs in the query_results + for y in range(0,len(query_results)): + if "doc_ID" in query_results[y]: + if query_results[y]["doc_ID"] not in doc_id_reviewed: + doc_id_reviewed.append(query_results[y]["doc_ID"]) + doc_fn_reviewed.append(query_results[y]["file_source"]) + + # second - identify and sort the key pages associated with the doc + for x in range(0,len(doc_id_reviewed)): + pages_reviewed = [] + for z in range(0,len(query_results)): + if "doc_ID" in query_results[z]: + if query_results[z]["doc_ID"] == doc_id_reviewed[x]: + pages_reviewed.append(query_results[z]["page_num"]) + + pr = Counter(pages_reviewed) + mc = pr.most_common() + page_output_list = [] + for m in mc: + page_output_list.append(m[0]) + + if len(doc_fn_reviewed) > x: + doc_fn_tmp = doc_fn_reviewed[x] + else: + doc_fn_tmp = "Doc# " + str(doc_id_reviewed[x]) + + bibliography.append({doc_fn_tmp:page_output_list}) + + return bibliography + + def filter_cursor_list(self, cursor, filter_dict, sample_count=20, exhaust_full_cursor=None): + + """ Applies filter to a cursor list. """ + + validated_filter_dict = self.prep_validated_filter_dict(filter_dict) + result_output = [] + + for i, entry in enumerate(cursor): + + for key, value in validated_filter_dict.items(): + if key not in entry: + logger.warning(f"warning: Query - retrieval cursor does not contain filter key - {key}") + else: + if entry[key] == value: + result_output.append(entry) + + if len(result_output) > sample_count and not exhaust_full_cursor: + break + + return result_output + + def prep_validated_filter_dict(self, filter_dict): + + """ Internal utility to prepare a validated filter dict. """ + + validated_filter_dict = {} + + for key, values in filter_dict.items(): + if key in self.library.default_keys: + validated_filter_dict.update({key:values}) + else: + logger.warning(f"warning: Query - filter key not in library collection - {key}") + + return validated_filter_dict + + def block_lookup_by_collection_id(self, _id): + + """ Block lookup using collection id key specifically. """ + + # specific to Mongo lookup - uses mongo '_id' which needs to be wrapped in ObjectId + return CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key("_id", ObjectId(_id)) + + def compare_text_blocks(self, t1, t2): + + """ Token-by-token comparison of two text blocks. """ + + b = CorpTokenizer(one_letter_removal=True, remove_numbers=True, remove_stop_words=True) + tokens1 = b.tokenize(t1) + tokens2 = b.tokenize(t2) + match_per = 0 + match = 0 + + for x in range(0, len(tokens1)): + for y in range(0, len(tokens2)): + if tokens1[x].lower() == tokens2[y].lower(): + match += 1 + break + + if len(tokens1) > 0: + match_per = match / len(tokens1) + + return match_per + + def block_similarity_retrieval_more_like_this (self, target_text, qr, similarity_threshold=0.25): + + """ Block similarity by token comparison. """ + + # will rank and order a list of query results using a target text as the reference point + output = [] + + for i, block in enumerate(qr): + + compare_text = block["text"] + similarity = self.compare_text_blocks(target_text, compare_text) + + if similarity > similarity_threshold: + block.update({"similarity": similarity}) + + output.append(block) + + output = sorted(output, key=lambda x:x["similarity"], reverse=True) + + return output + + def build_doc_id_fn_list(self, qr): + + """ Utility function to build a doc_id and filename list. """ + doc_id_list = [] + fn_list = [] + + for q in qr: + if q["doc_ID"] not in doc_id_list: + doc_id_list.append(q["doc_ID"]) + fn_list.append(q["file_source"]) + + return doc_id_list, fn_list + + def expand_text_result_before(self, block, window_size=400): + + """ Expands text result before. """ + + block_id = block["block_ID"] -1 + doc_id = block["doc_ID"] + + before_text = "" + pre_blocks = [] + + while len(before_text) < window_size and block_id >= 0: + + before_block = self.block_lookup(block_id, doc_id) + + if before_block: + before_text += before_block["text"] + pre_blocks.append(before_block) + + output = {"expanded_text": before_text, "results": pre_blocks} + + return output + + def expand_text_result_after(self, block, window_size=400): + + """ Expands text result after. """ + + block_id = block["block_ID"] + 1 + doc_id = block["doc_ID"] + + after_text = "" + post_blocks = [] + + while len(after_text) < window_size: + after_block = self.block_lookup(block_id, doc_id) + if not after_block: + break # Break if no block is found + + after_text += after_block["text"] + post_blocks.append(after_block) + block_id += 1 # Increment block_id for next iteration + + output = {"expanded_text": after_text, "results": post_blocks} + return output + + def generate_csv_report(self): + + """Generates a csv report from the current query status. """ + + output = QueryState(self).generate_query_report_current_state() + return output + + def filter_by_key_value_range(self, key, value_range, results_only=True): + + """ Executes a filter by key value range. """ + + cursor = CollectionRetrieval(self.library_name, + account_name=self.account_name).filter_by_key_value_range(key,value_range) + + query= "" + result_dict = self._cursor_to_qr(query, cursor, exhaust_full_cursor=True) + + if results_only: + return result_dict["results"] + + return result_dict + diff --git a/llmware/setup.py b/llmware/setup.py new file mode 100644 index 0000000..646e27a --- /dev/null +++ b/llmware/setup.py @@ -0,0 +1,179 @@ +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + +"""The setup module implements the init process. + +The module implements the Setup class, which has two static methods - load_sample_files and load_sample_voice_files. + +These methods create the necessary directory if they do not exist and downloads the sample files from an llmware- + maintained AWS S3 instance. + +""" + + +import shutil +import os +from llmware.resources import CloudBucketManager +from llmware.configs import LLMWareConfig + +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(level=LLMWareConfig().get_logging_level_by_module(__name__)) + + +class Setup: + + """Implements the download of sample files from an AWS S3 bucket. + + ``Setup`` implements the download of sample files from an AWS S3 bucket. Currently, there are samples + from eight domains: + + - AgreementsLarge (~80 sample contracts) + - Agreements (~15 sample employment agreements) + - UN-Resolutions-500 (500 United Nations Resolutions over ~2 years) + - Invoices (~40 invoice sample documents) + - FinDocs (~15 financial annual reports, earnings and 10Ks) + - AWS-Transcribe (~5 AWS-transcribe JSON files) + - SmallLibrary (~10 mixed document types for quick testing) + - Images (~3 images for OCR processing) + + The sample files are updated continously. By calling ``Setup().load_sample_files(over_write=True)`` + you will get the newest version of the sample files. + + The sample files were prepared by LLMWare from public domain materials, or invented bespoke. + If you have any concerns about Personally Identifiable Information (PII), or the suitability of any material + we included, please contact us, e.g. either by raising an issue on GitHub or sending an E-Mail. + We reserve the right to withdraw documents at any time. + + Examples + ---------- + >>> import os + >>> from llmware.setup import Setup + >>> sample_files_path = Setup().load_sample_files() + >>> sample_files_path + '/home/user/llmware_data/sample_files' + >>> os.listdir(sample_files_path) + ['AWS-Transcribe', '.DS_Store', 'SmallLibrary', 'UN-Resolutions-500', 'Invoices', 'Images', 'AgreementsLarge', 'Agreements', 'FinDocs'] + + If you have called the function before but want to get the newest updates to the sample files, or you simply + want to get the newest sample files, you simply set ``over_write=True``. + >>> sample_files_path = Setup().load_sample_files(over_write=True) + """ + @staticmethod + def load_sample_files(over_write=False): + + """ Downloads sample document files from non-restricted AWS S3 bucket. """ + + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + + # not configurable - will pull into /sample_files under llmware_path + sample_files_path = os.path.join(LLMWareConfig.get_llmware_path(), "sample_files") + + if not os.path.exists(sample_files_path): + os.makedirs(sample_files_path,exist_ok=True) + else: + if not over_write: + logger.info(f"Setup - sample_files path already exists - {sample_files_path}") + return sample_files_path + + # pull from sample files bucket + logger.info(f"Setup - sample_files - downloading requested sample files from AWS S3 bucket - may take a minute.") + + bucket_name = LLMWareConfig().get_config("llmware_sample_files_bucket") + remote_zip = bucket_name + ".zip" + local_zip = os.path.join(sample_files_path, bucket_name + ".zip") + + CloudBucketManager().pull_file_from_public_s3(remote_zip, local_zip, bucket_name) + shutil.unpack_archive(local_zip, sample_files_path, "zip") + os.remove(local_zip) + + return sample_files_path + + @staticmethod + def load_voice_sample_files(over_write=False, small_only=True): + + """ Downloads sample wav files from non-restricted AWS S3 bucket. """ + + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + + # not configurable - will pull into /sample_files under llmware_path + if not small_only: + sample_files_path = os.path.join(LLMWareConfig.get_llmware_path(), "voice_sample_files") + else: + sample_files_path = os.path.join(LLMWareConfig.get_llmware_path(), "voice_sample_files_small") + + if not os.path.exists(sample_files_path): + os.makedirs(sample_files_path, exist_ok=True) + else: + if not over_write: + logger.info(f"Setup - voice_sample_files path already exists - {sample_files_path}") + return sample_files_path + + # pull from sample files bucket + bucket_name = LLMWareConfig().get_config("llmware_sample_files_bucket") + + if small_only: + folder_name = "voice_small" + else: + folder_name = "voice_all" + + logger.info("Setup - sample_voice_files - downloading requested sample files from AW3 S3 bucket - " + "may take a minute.") + + remote_zip = folder_name + ".zip" + local_zip = os.path.join(sample_files_path, bucket_name + ".zip") + + CloudBucketManager().pull_file_from_public_s3(remote_zip, local_zip, bucket_name) + shutil.unpack_archive(local_zip, sample_files_path, "zip") + os.remove(local_zip) + + return sample_files_path + + @staticmethod + def load_selected_sample_files(sample_folder="microsoft_ir", over_write=False): + + """ Downloads sample wav files from non-restricted AWS S3 bucket. """ + + if not os.path.exists(LLMWareConfig.get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + + # not configurable - will pull into /sample_files under llmware_path + sample_files_path = os.path.join(LLMWareConfig.get_llmware_path(), sample_folder) + + if not os.path.exists(sample_files_path): + os.makedirs(sample_files_path, exist_ok=True) + else: + if not over_write: + logger.info(f"Setup - sample_files selected path already exists - {sample_files_path}") + return sample_files_path + + # pull from sample files bucket + bucket_name = LLMWareConfig().get_config("llmware_sample_files_bucket") + + folder_name = sample_folder + + logger.info("Setup - selected sample files - downloading requested sample files from AW3 S3 bucket - " + "may take a minute.") + + remote_zip = folder_name + ".zip" + local_zip = os.path.join(sample_files_path, bucket_name + ".zip") + + CloudBucketManager().pull_file_from_public_s3(remote_zip, local_zip, bucket_name) + shutil.unpack_archive(local_zip, sample_files_path, "zip") + os.remove(local_zip) + + return sample_files_path diff --git a/llmware/util.py b/llmware/util.py new file mode 100755 index 0000000..b6299c0 --- /dev/null +++ b/llmware/util.py @@ -0,0 +1,2246 @@ + +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + + +"""The util module implements general helper functions that are used across LLMWare, primarily within the Utilities +class, along with a whole word (white space) tokenizer (CorpTokenizer) class, TextChunker and AgentWriter classes. """ + + +import csv +from collections import Counter +import sys +import os +import random + +import platform +from pathlib import Path +import re +from datetime import datetime +from ctypes import * +import shutil + +import logging + +from llmware.resources import CloudBucketManager +from llmware.configs import (LLMWareConfig, LLMWareException, ModuleNotFoundException, + DependencyNotInstalledException, ModelNotFoundException) + + +try: + from tokenizers import Tokenizer +except: + logging.warning("tokenizers library could not be imported - some functionality may not be available.\n" + "to fix: pip3 install tokenizers") + tokenizers = None + +logger = logging.getLogger(__name__) + + +class Utilities: + + """ Utility functions used throughout LLMWare """ + + def __init__(self, library=None): + self.library = library + + def get_module_pdf_parser(self): + + """ Loads shared libraries for the Parser module, based on machine architecture. """ + + # Detect machine architecture + if platform.system() == "Windows": + system = "windows" + file_ext = ".dll" + + if platform.machine().lower() == "arm64": + machine = "arm64" + if LLMWareConfig().get_active_db() != "sqlite": + logger.warning(f"Currently Windows Arm64 parser only supports SQLite. Automatically " + f"changing active db setting to SQLite.") + LLMWareConfig().set_active_db("sqlite") + + else: + machine = "x86_64" + else: + system = platform.system().lower() + machine = os.uname().machine.lower() + file_ext = ".so" + + # Default to known architectures if we encounter an unknown one + if system == 'darwin' and machine not in ['arm64', 'x86_64']: + machine = 'arm64' + if system == 'linux' and machine not in ['aarch64', 'x86_64']: + machine = 'x86_64' + + if system == 'linux' and machine == 'aarch64': + + """ 0.4.4 - aarch64 linux in process of being supported + -- re-integrating parsers on aarch64 linux + -- removing deprecation warnings """ + + pass + + # deprecation warning for darwin x86_64 + if system == "darwin" and machine == "x86_64": + + error_msg = ("Mac x86 detected as OS - this is not a supported platform. Support " + "was deprecated in llmware version 0.2.6 and removed in llmware version 0.3.9. " + "Options - move to Mac Metal (M1+), back-level llmware to supported version, or " + "if urgent requirement for Mac x86, please raise ticket on github.") + + raise LLMWareException(message=error_msg) + + machine_dependent_lib_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), system, machine) + _path_pdf = os.path.join(machine_dependent_lib_path, "llmware", "libpdf_llmware" + file_ext) + + _mod_pdf = None + + try: + # attempt to load the shared library with ctypes + _mod_pdf = cdll.LoadLibrary(_path_pdf) + + except: + # catch error, if possible + logger.warning(f"Module 'PDF Parser' could not be loaded from path - \n {_path_pdf}.\n") + + # if no module loaded, then raise exception + if not _mod_pdf: + raise ModuleNotFoundException("PDF Parser") + + return _mod_pdf + + def get_module_office_parser(self): + + """ Load shared libraries for Office parser module based on machine architecture. """ + + # Detect machine architecture + if platform.system() == "Windows": + system = "windows" + file_ext = ".dll" + + if platform.machine().lower() == "arm64": + machine = "arm64" + + if LLMWareConfig().get_active_db() != "sqlite": + logger.warning(f"Currently Windows Arm64 parser only supports SQLite. Automatically " + f"changing active db setting to SQLite.") + LLMWareConfig().set_active_db("sqlite") + + else: + machine = "x86_64" + else: + system = platform.system().lower() + machine = os.uname().machine.lower() + file_ext = ".so" + + # Default to known architectures if we encounter an unknown one + if system == 'darwin' and machine not in ['arm64', 'x86_64']: + machine = 'arm64' + if system == 'linux' and machine not in ['aarch64', 'x86_64']: + machine = 'x86_64' + + if system == 'linux' and machine == 'aarch64': + + """ 0.4.4 - aarch64 linux in process of being supported + -- re-integrating parsers on aarch64 linux + -- removing deprecation warnings """ + + pass + + # deprecation warning for darwin x86_64 + if system == "darwin" and machine == "x86_64": + + error_msg = ("Mac x86 detected as OS - this is not a supported platform. Support " + "was deprecated in llmware version 0.2.6 and removed in llmware version 0.3.9. " + "Options - move to Mac Metal (M1+), back-level llmware to supported version, or " + "if urgent requirement for Mac x86, please raise ticket on github.") + + raise LLMWareException(message=error_msg) + + machine_dependent_lib_path = os.path.join(LLMWareConfig.get_config("shared_lib_path"), system, machine) + _path_office = os.path.join(machine_dependent_lib_path, "llmware", "liboffice_llmware" + file_ext) + + _mod = None + + try: + # attempt to load the shared library with ctypes + _mod = cdll.LoadLibrary(_path_office) + except: + # catch the error, if possible + logger.warning(f"Module 'Office Parser' could not be loaded from path - \n {_path_office}.\n") + + # if no module loaded, then raise exception + if not _mod: + raise ModuleNotFoundException("Office Parser") + + return _mod + + def get_default_tokenizer(self): + + """ Retrieves an instance of default tokenizer. In most cases, this is the GPT2 tokenizer, which is a + good proxy for OpenAI and OpenAI-like GPTNeo models. """ + + # gpt2 tokenizer is used in several places as a default tokenizer + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + # first, check if it is in the local repo + local_model_repo = LLMWareConfig.get_model_repo_path() + models = os.listdir(local_model_repo) + + if "gpt2" not in models: + + # if not found locally, then pull from global repo + + logger.info("Utilities - get_default_tokenizer - if no tokenizer found, then as a backup, " + "the gpt2 tokenizer will be used - not in local model repository, " + "so pulling from global repo - this may take a few seconds the first time to download.") + + files = CloudBucketManager().pull_single_model_from_llmware_public_repo(model_name="gpt2") + + # quick check to confirm that model is present + models = os.listdir(local_model_repo) + if "gpt2" not in models: + raise ModelNotFoundException("gpt2_tokenizer") + + tokenizer = Tokenizer.from_file(os.path.join(local_model_repo, "gpt2", "tokenizer.json")) + + return tokenizer + + def load_tokenizer_from_file(self, fp): + + """ Loads tokenizer from file. """ + + tokenizer = Tokenizer.from_file(fp) + return tokenizer + + def get_uuid(self): + + """ Generates a UUID. """ + + import uuid + # uses unique id creator from uuid library + return uuid.uuid4() + + @staticmethod + def file_save (cfile, file_path, file_name): + + """ Saves an in-memory array to CSV file. """ + + max_csv_size = 20000 + csv.field_size_limit(max_csv_size) + + out_file = os.path.join(file_path, file_name) + + with open(out_file, 'w', newline='') as csvfile: + c = csv.writer(csvfile, dialect='excel', doublequote=False, delimiter=',',escapechar = ']') + + for z in range(0, len(cfile)): + # intercept a line too large here + if sys.getsizeof(cfile[z]) < max_csv_size: + try: + # unusual, but if unable to write a particular element, then will catch error and skip + c.writerow(cfile[z]) + except: + logger.warning(f"File save - could not write item in row {z} - skipping") + pass + else: + logger.error(f"CSV ERROR: Row exceeds MAX SIZE: {sys.getsizeof(cfile[z])} - " + f"{cfile[z]}") + + csvfile.close() + + return 0 + + @staticmethod + def file_load (in_path, delimiter=",",encoding='ISO-8859-1',errors='ignore'): + + """ Loads a CSV array and outputs an in-memory array corresponding to the CSV structure. """ + + record_file = open(in_path, encoding=encoding,errors=errors) + c = csv.reader(record_file, dialect='excel', doublequote=False, delimiter=delimiter) + output = [] + for lines in c: + output.append(lines) + record_file.close() + + return output + + @staticmethod + def csv_save(rows, file_dir, file_name): + + """ Saves CSV from in memory array consisting of list of rows as input. """ + + full_path = Path(file_dir, file_name) + + with full_path.open('w', encoding='utf-8') as out: + writer = csv.writer(out) + try: + writer.writerows(rows) + except csv.Error as e: + logger.error("Exception writing csv file - not successful.") + return False + + return True + + @staticmethod + def get_top_bigrams (tokens, top_n): + + """ Returns a list of top_n bigrams based on a list of tokens. """ + + bigrams = [] + for z in range(1, len(tokens)): + entry = (tokens[z-1] + "_" + tokens[z]) + bigrams.append(entry) + + d = Counter(bigrams) + dc = d.most_common(top_n) + + return dc + + @staticmethod + def get_top_trigrams (tokens, top_n): + + """ Returns a list of top_n trigrams based on a list of tokens. """ + + trigrams = [] + for z in range(2 ,len(tokens)): + entry = (tokens[ z -2] + "_" + tokens[ z -1] + "_" + tokens[z]) + trigrams.append(entry) + + d = Counter(trigrams) + dc = d.most_common(top_n) + + return dc + + @staticmethod + def get_top_4grams (tokens, top_n): + + """ Returns a list of top_n 4grams based on a list of tokens. """ + + four_grams = [] + for z in range(3 ,len(tokens)): + entry = (tokens[ z -3 ]+ "_" + tokens[ z -2] + "_" + tokens[ z -1] + "_" + tokens[z]) + four_grams.append(entry) + + d = Counter(four_grams) + dc = d.most_common(top_n) + + return dc + + @staticmethod + def compare_timestamps (t1, t2, time_str="%a %b %d %H:%M:%S %Y"): + + """ Compares two time-stamps t1 and t2 provided as input and returns a time_delta_obj, along + with explicitly passing the days and seconds from the time_delta_obj. """ + + t1_obj = datetime.strptime(t1, time_str) + t2_obj = datetime.strptime(t2, time_str) + + time_delta_obj = t1_obj - t2_obj + + days = time_delta_obj.days + seconds = time_delta_obj.seconds + + return time_delta_obj, days, seconds + + @staticmethod + def get_current_time_now (time_str="%a %b %e %H:%M:%S %Y"): + + """ Returns the current time, used for time-stamps - delivered in format from the optional input + time_str. """ + + # if time stamp is used in file_name, needs to be Windows standards compliant + if platform.system() == "Windows": + time_str = "%Y-%m-%d_%H%M%S" + + return datetime.now().strftime(time_str) + + @staticmethod + def get_time_string_standard(): + + """ Returns the time stamp string standard used. """ + + time_str_standard = "%a %b %e %H:%M:%S %Y" + return time_str_standard + + @staticmethod + def isfloat(num): + + """ Checks if an input is a float number. """ + + try: + float(num) + return True + except ValueError: + return False + + @staticmethod + def prep_filename_alt(filename_in, accepted_file_formats_list): + + """ Prepares a filename and offers options to configure and provide safety checks to provide a 'safe' + filename. """ + + success_code = 1 + + fn_toks = filename_in.split(".") + fn_base = fn_toks[0] + ext = fn_toks[-1] + + # only accept upload files with file extension in accepted_file_formats_list + if ext.lower() in accepted_file_formats_list and not filename_in.startswith("."): + + # prepend a random number to the front of the secure filename + + if len(fn_base) > 240: + # cap len of filename at 240 + filename_in = fn_base[0:240] + "." + ext + + fn_out = str(random.randint(100000, 999999)) + "_" + filename_in + + else: + success_code = -1 + fn_out = filename_in + + return success_code, fn_out + + @staticmethod + def safe_url(string): + + """ Confirms that a string is a safe url. """ + + try: + import urllib.parse + return urllib.parse.quote_plus(string) + except TypeError: + logger.error(f"Error encoding string - {string}") + return "" + + def prune_stop_words(self, text,front=100,back=100): + + """ Utility function that strips stop words from context text, with + the goal of reducing context size, while keeping semantic meaning in + place - intended for use in large contexts run in smaller memory space. """ + + stripped_text = "" + stop_words = Utilities().get_stop_words_master_list() + tokens = text.split(" ") + front_reserve = front + back_reserve = len(tokens) - back + word_reduction = 0 + + for i, tok in enumerate(tokens): + + if front_reserve < i < back_reserve: + if tok.lower() in stop_words: + word_reduction += 1 + pass + else: + stripped_text += tok + " " + else: + stripped_text += tok + " " + + logger.info(f"Utilities - prune_stop_words - {stripped_text}") + logger.info(f"Utilities - prune_stop_words - " + f"word reduction - {word_reduction}") + + return stripped_text + + @staticmethod + def get_stop_words_master_list(): + + """ Returns a common set of english stop words. """ + + stop_words = ["a", "able", "about","above","accordance","according", "accordingly","across","act","actually", + "added" ,"adj" ,"affected" ,"affecting" ,"affects" ,"after" ,"afterwards" ,"again" ,"against", + "ah","al" ,"all", "almost" ,"alone" ,"along" ,"already", "also" ,"although" ,"always" ,"am" , + "among" ,"amongst" ,"an","and","announce" ,"another" ,"any" ,"anybody" ,"anyhow" ,"anymore" , + "anyone" ,"anything" ,"anyway","anyways","anywhere" ,"apparently" ,"approximately" ,"are" , + "aren" ,"arent" ,"arise", "around", "as" ,"aside", "ask", "asked" ,"asking" ,"at" ,"auth", + "available" ,"away" ,"awfully" ,"b" ,"back", "basically" ,"be", "became" ,"because", + "become" ,"becomes", "becoming" ,"been", "before" ,"beforehand", "begin", "beginning" ,"beginnings", + "begins" ,"behind" ,"being" ,"believe" ,"below" ,"beside" ,"besides" ,"between" ,"beyond", "biol" + ,"both", "brief" ,"briefly" ,"but" ,"by" ,"c" ,"ca" ,"came" ,"can" ,"cannot" ,"can't" ,"cant" ,"cause" + ,"causes", "certain" ,"certainly" ,"co" ,"com" ,"come" ,"comes" ,"contain" ,"containing" ,"contains", + "could","couldnt", "d","did" ,"didnt" ,"didn't", "different" ,"do" ,"does" ,"doesn't", + "doesnt" ,"doing","done","don't" ,"dont" ,"down" ,"downwards" ,"due" ,"during" ,"e" ,"each" , + "ed","edu","effect","eg","e.g." ,"eight", "eighty" ,"either" ,"else" ,"elsewhere" ,"end" , + "ending" ,"enough" ,"especially" ,"et" ,"etal" ,"etc" ,"even","ever" ,"every" ,"everybody", + "everyone" ,"everything" ,"everywhere" ,"ex" ,"except" ,"f" ,"far" ,"few" ,"ff", "fifth", + "first" ,"five" ,"fix" ,"followed" ,"following" ,"follows" ,"for" ,"former" ,"formerly","forth", + "found" ,"four" ,"from" ,"further" ,"furthermore" ,"g" ,"gave" ,"generally" ,"get" ,"gets" ,"getting" + ,"give" ,"given", "gives" ,"giving" ,"go" ,"goes" ,"gone" ,"got" ,"gotten" ,"h" ,"had" ,"happens", + "hardly" ,"has","hasn't","have" ,"haven't" ,"having" ,"he" ,"hed" ,"hence" ,"her" ,"here", + "hereafter" ,"hereby" ,"herein","heres", "here's" ,"hereupon" ,"hers" ,"herself" ,"hes" ,"he's", + "hi" ,"hid" ,"him" ,"himself" ,"his" ,"hither" ,"home", "how" ,"howbeit" ,"however" ,"hundred", + "i" ,"id" ,"ie" ,"i.e." ,"if" ,"i'll" ,"ill" ,"im" ,"i'm" ,"immediate", "immediately" ,"importance", + "important" ,"in" ,"inc" ,"inc." ,"indeed" ,"index" ,"information","instead", "into", + "invention","inward" ,"is" ,"isn't" ,"isnt" ,"it" ,"itd" ,"it'll","its","it's" ,"itself" + ,"i've" ,"ive" ,"j", "just" ,"k" ,"keep" ,"keeps" ,"kept" ,"kg" ,"km" ,"know","known","knows", + "l","largely","last","lately", "later","latter","latterly","least","less","lest","let","lets", + "let's" ,"like" ,"liked","likely", "line" ,"little" ,"'ll" ,"look" ,"looking" ,"looks", + "ltd" ,"m" ,"made" ,"mainly" ,"make" ,"makes","many", "may" ,"maybe" ,"me" ,"mean" ,"means" , + "meantime" ,"meanwhile" ,"merely" ,"mg" ,"might","miss", "ml" ,"more" ,"moreover", + "most" ,"mostly" ,"mr" ,"mr." ,"mrs" ,"mrs." ,"ms", "ms." ,"much" ,"mug","must" ,"my" ,"myself", + "n" ,"na" ,"name" ,"namely" ,"nay" ,"nd" ,"near" ,"nearly" ,"necessarily" ,"necessary" ,"need" + ,"needs", "neither" ,"never""nevertheless" ,"new" ,"next" ,"nine" ,"ninety" ,"no" ,"nobody", + "non" ,"none","nonetheless","noone" ,"nor" ,"normally" ,"nos" ,"not" ,"note" ,"noted" , + "nothing" ,"now" ,"nowhere" ,"o" ,"obtain","obtained", "obviously" ,"of" ,"off" ,"often", + "oh" ,"ok" ,"okay" ,"old" ,"omit" ,"omitted" ,"on" ,"once" ,"one","ones","only" ,"onto" ,"or", + "ord" ,"other" ,"others" ,"otherwise" ,"ought" ,"our" ,"ours" ,"ourselves","out", + "outside" ,"over" ,"overall" ,"owing" ,"own" ,"p" ,"page" ,"pages" ,"part" ,"particular" + ,"particularly", "past" ,"per" ,"perhaps" ,"placed" ,"please" ,"plus" ,"poorly" ,"possible", + "possibly" ,"potentially","pp","predominantly" ,"present" ,"previously" ,"primarily","probably", + "promptly" ,"proud" ,"provide", "provides" ,"put" ,"q" ,"que" ,"quickly" ,"quite" ,"qv" , + "r" ,"ran" ,"rather" ,"rd" ,"re" ,"readily","really","recent" ,"recently" ,"ref" ,"refs", + "regarding" ,"regardless" ,"regards" ,"regard" ,"related","relative", "relatively" , + "research","respectively" ,"resulted" ,"resulting", "right" ,"run" ,"s","said", + "same" ,"saw" ,"say" ,"saying" ,"says" ,"see" ,"seeing" ,"seem" ,"seemed","seeming","seems", + "seen" ,"self","selves" ,"sent" ,"seven" ,"several" ,"shall" ,"she" ,"shed" ,"she'll" ,"shes", + "she's" ,"should","shouldn't", "shouldnt" ,"show" ,"showed" ,"shown" ,"showns" ,"shows" , + "significant" ,"significantly" ,"similar", "similarly" ,"since" ,"six" ,"slightly" ,"so" , + "some" ,"somebody" ,"somehow" ,"someone","something" ,"sometime" ,"sometimes" , + "somewhat" ,"somewhere" ,"soon" ,"sorry" ,"specifically","specified", "specify" , + "specifying" ,"still" ,"stop" ,"strongly" ,"sub" ,"substantially" ,"successfully" ,"such", + "sufficiently" ,"suggest" ,"sup" ,"sure" ,"t" ,"take" ,"taken" ,"taking" ,"talk" , + "talked" ,"td","tell" ,"tends" ,"th" ,"than", "thank" ,"thanks" ,"thanx" ,"that" ,"that'll" , + "thats" ,"that've" ,"the" ,"their" ,"theirs" ,"them", "themselves" ,"then" ,"thence" , + "there" ,"thereafter", "thereby" ,"thered" ,"therefore" ,"therein","there'll" ,"thereof", + "therere" ,"theres" ,"thereto" ,"thereupon" ,"there've" ,"these", "they", + "theyd" ,"they'll" ,"theyre" ,"they've" ,"think" ,"this" ,"those" ,"thou" ,"though" ,"thoughh" + ,"thousand", "throug" ,"through" ,"throughout" ,"thru" ,"thus" ,"til" ,"tip" ,"to" , + "together" ,"too" ,"took","toward","towards" ,"tr" ,"tried" ,"tries" ,"truly" ,"try" , + "trying" ,"ts" ,"twice" ,"two", "u" ,"un" ,"under", "unfortunately" ,"unless" ,"unlike" , + "unlikely" ,"until" ,"unto" ,"up" ,"upon" ,"ups" ,"us" ,"use","used","useful", + "usefully" ,"usefulness" ,"uses" ,"using" ,"usually" ,"v" ,"value" ,"various" ,"ve" ,"very" + ,"via","viz" ,"vol" ,"vols" ,"vs" ,"w" ,"want" ,"wants" ,"was" ,"wasnt" ,"way" , + "we" ,"wed" ,"welcome","well" ,"we'll" ,"went","were" ,"werent" ,"we've" ,"what" ,"whatever", + "what'll" ,"whats" ,"when" ,"whence" ,"whenever","where","whereafter", "whereas", + "whereby" ,"wherein" ,"wheres" ,"whereupon" ,"wherever" ,"whether" ,"which", + "while" ,"whim" ,"whither" ,"who" ,"whod" ,"whoever" ,"whole" ,"who'll" ,"whom","whomever","whos" + ,"whose", "why" ,"widely" ,"willing" ,"will" ,"wish" ,"with" ,"within" ,"without","wont", + "words" ,"world" ,"would" ,"wouldnt","www" ,"x" ,"xx" ,"xxx", "y" ,"yes" ,"yet" , + "you" ,"youd" ,"you'll" ,"your" ,"youre" ,"yours" ,"yourself","yourselves" ,"you've" ,"z", + "zero" ,"xoxo", "ii", "iii", "iv" ,"ix" ,"vi" ,"vii" ,"viii" ,"", + "" ,"three" ,"ten" ,"view" ,"met" ,"follow" ,"consist" ,"lack" ,"lacks","based" ,"ago", + "addition" ,"additional" ,"depend" ,"depends" ,"include" ,"includes" ,"including" ,"continue" + ,"bring", "brings" ,"ahead" ,"add" ,"adds" ,"attribute" ,"attributes" ,"associated" ,"associate", "follow", + "happen" ,"happened" ,"happening" ,"single" ,"consider" ,"considered" ,"looked" ,"involve" + ,"involves", "involved" ,"thing" ,"things" ,"going", "brought", "lot"] + + return stop_words + + def load_stop_words_list (self, library_fp): + + """ Loads a stop words list from file. """ + + stop_words = self.get_stop_words_master_list() + + s = open(os.path.join(library_fp, "stop_words_list.txt"), "w", encoding='utf-8') + + for words in stop_words: + s.write((words + ",")) + s.close() + os.chmod((library_fp+ "stop_words_list.txt"), 0o777) + + return stop_words + + def remove_stop_words(self, token_list): + + """ Filters a list of tokens and removes stop words. """ + + stop_words = self.get_stop_words_master_list() + + tokens_out = [] + for z in range(0, len(token_list)): + if token_list[z] not in stop_words: + tokens_out.append(token_list[z]) + + return tokens_out + + @staticmethod + def clean_list (token_list): + + """ Used by CorpTokenizer to provide a clean list stripping punctuation. """ + + punctuation = ("-" ,"," ,"'", "/" ,"(')", "'('" ,":" ,".", "?" ,"%", "[", "]" ,"(')'" ,"('('" ,"'–'", ";") + clean_out = [] + for z in range(0 ,len(token_list)): + t = token_list[z] + clean_word = "" + for y in range(0 ,len(t)): + if t[y] in punctuation: + if len(clean_word) == len(t) -1: + # if last letter in word, then skip, no additional space added + char_out = "" + else: + char_out = "" + else: + char_out = t[y] + clean_word += char_out + + if clean_word != "": + clean_out.append(clean_word) + + return clean_out + + def sentence_splitter(self, sentence, key_word, marker_list): + + """ Splits a sentence around a marker word. """ + + text = [] + completion = [] + # will split sentence either 'before' or 'after' the marker + # simplest pattern - split at marker + + for m in marker_list: + + # if key_word is at the start of the sentence, e.g., marker = 0, include in text ... + if m < len(key_word): + text.append(sentence[0:m+len(key_word)]) + completion.append(sentence[m+len(key_word):]) + else: + text.append(sentence[0:m]) + completion.append(sentence[m:]) + + return text, completion + + def prep_custom_mlm_label (self, input_sentence,key_word_list, mask_token_value="", mlm_prob=0.15): + + """ Prepares a custom masked language label. """ + + label_id = [] + for x in input_sentence: + r = random.randint(1,100) + if r <= (mlm_prob * 100): + r2 = random.randint(1,10) + if r2 <= 10: + label_id.append(mask_token_value) + else: + # keep original value + label_id.append(x) + + return label_id + + def exact_search_dicts(self, query, output_dicts, text_key="text",remove_stop_words=True, + mode="or"): + + """ Executes a fast 'lightweight' in-memory token search across a list of dictionaries + + -- query: filtering query - looking for an exact phrase + -- output_dicts: can be any list of dicts provided that the text_key is found in the dict + -- text_key: by default, this is "text", but can be configured to any field in the dict + -- remove_stop_words: set to True by default + + Returns a subset of the list of the dicts with only those entries that match the query + """ + + matched_dicts = [] + + # handle edge case - if empty search result, then return all dicts with updated keys + if not query: + for i, entries in enumerate(output_dicts): + if "page_num" not in entries: + if "master_index" in entries: + page_num = entries["master_index"] + else: + page_num = 0 + entries.update({"page_num": page_num}) + if "query" not in entries: + entries.update({"query": ""}) + matched_dicts.append(entries) + return matched_dicts + + for i, entries in enumerate(output_dicts): + + if query.lower() in entries[text_key].lower(): + + if "page_num" not in entries: + if "master_index" in entries: + page_num = entries["master_index"] + else: + page_num = 0 + + entries.update({"page_num": page_num}) + + if "query" not in entries: + entries.update({"query": query}) + + matched_dicts.append(entries) + + return matched_dicts + + def token_search_dicts(self, query, output_dicts, text_key="text",remove_stop_words=True, + mode="or"): + + """ Executes a fast 'lightweight' in-memory token search across a list of dictionaries + + -- query: filtering query - tokenized + -- output_dicts: can be any list of dicts provided that the text_key is found in the dict + -- text_key: by default, this is "text", but can be configured to any field in the dict + -- remove_stop_words: set to True by default + -- mode: set to either logical 'or' or 'and' + -- if 'or', then will return any entry with one of the matching tokens in the query. + -- if 'and', then will return entry only if it contains all tokens in the query. + + Returns a subset of the list of the dicts with only those entries that match the query + """ + + matched_dicts = [] + + c = CorpTokenizer(remove_stop_words=remove_stop_words, remove_numbers=False, one_letter_removal=True, + remove_punctuation=True) + + key_terms = c.tokenize(query) + + # handle edge case - if empty search result, then return all dicts with updated keys + if len(key_terms) == 0: + for i, entries in enumerate(output_dicts): + if "page_num" not in entries: + if "master_index" in entries: + page_num = entries["master_index"] + else: + page_num = 0 + entries.update({"page_num": page_num}) + if "query" not in entries: + entries.update({"query": ""}) + matched_dicts.append(entries) + return matched_dicts + + # len of key_terms >= 1 -> initiate key term match search + for i, entries in enumerate(output_dicts): + text_tokens = c.tokenize(entries[text_key]) + match = 0 + keep = False + + for j, tok in enumerate(key_terms): + + # match of token with text + if tok in text_tokens: + match += 1 + if mode == "or": + keep = True + break + + # strip trailing 's' and look for match + elif tok.endswith("s"): + if tok[:-1] in text_tokens: + match += 1 + if mode == "or": + keep = True + break + + # append trailing 's' and look for match + elif (tok+"s") in text_tokens: + match += 1 + if mode == "or": + keep = True + break + + if mode == "and" and match == len(key_terms): + keep = True + + if keep: + if "page_num" not in entries: + if "master_index" in entries: + page_num = entries["master_index"] + else: + page_num = 0 + + entries.update({"page_num": page_num}) + + if "query" not in entries: + entries.update({"query": query}) + + matched_dicts.append(entries) + + return matched_dicts + + def fast_search_dicts(self, query,output_dicts, text_key="text", remove_stop_words=True): + + """ Executes a fast in-memory exact search across a list of dictionaries + + -- query: filtering query (exact match) + -- output_dicts: can be any list of dicts provided that the text_key is found in the dict + -- text_key: by default, this is "text", but can be configured to any field in the dict + -- remove_stop_words: set to True by default. + + Returns a subset of the list of the dicts with only those entries that match the query + """ + + # will return a subset of the output_dicts that have the key_terms + # no ranking or prioritization - "match" or "no-match" only + # designed primarily to filter in-memory sources and parser outputs + + matched_dicts = [] + + c = CorpTokenizer(remove_stop_words=remove_stop_words, remove_numbers=False, one_letter_removal=True, + remove_punctuation=True) + + key_terms = c.tokenize(query) + + # handle edge case - if empty search result, then return all dicts with updated keys + if len(key_terms) == 0: + for i, entries in enumerate(output_dicts): + if "page_num" not in entries: + if "master_index" in entries: + page_num = entries["master_index"] + else: + page_num = 0 + entries.update({"page_num": page_num}) + if "query" not in entries: + entries.update({"query": ""}) + matched_dicts.append(entries) + return matched_dicts + + # len of key_terms >= 1 -> initiate key term match search + for i, entries in enumerate(output_dicts): + + text_tokens = c.tokenize(entries[text_key]) + + for j, toks in enumerate(text_tokens): + match_found = 0 + if toks.lower() == key_terms[0].lower(): + match_found += 1 + + if len(key_terms) > 1: + if len(text_tokens) > (j + len(key_terms)): + for x in range(1,len(key_terms)): + if text_tokens[j+x].lower() == key_terms[x].lower(): + match_found += 1 + else: + match_found = 0 + break + + if match_found == len(key_terms): + # found confirmed match + + if "page_num" not in entries: + + if "master_index" in entries: + page_num = entries["master_index"] + else: + page_num = 0 + + entries.update({"page_num": page_num}) + + if "query" not in entries: + entries.update({"query": query}) + + matched_dicts.append(entries) + break + + return matched_dicts + + def find_match(self, key_term, sentence): + + """ Utility method that runs search for key_term in sentence. """ + + matches_found = [] + for x in range(0,len(sentence)): + match = 0 + if sentence[x].lower() == key_term[0].lower(): + match += 1 + if (x+len(key_term)) <= len(sentence): + for y in range(1,len(key_term)): + if key_term[y].lower() == sentence[x+y].lower(): + match += 1 + else: + match = -1 + break + + if match == len(key_term): + matches_found.append(x) + + return matches_found + + def locate_query_match(self, query, core_text): + + """ Utility function to locate character-level match of a query inside a core_text. """ + + import re + matches_found = [] + if not query: + return matches_found + + # tokenize the query + + b = CorpTokenizer(one_letter_removal=True, remove_stop_words=True, remove_punctuation=True, + remove_numbers=False) + + query_tokens = b.tokenize(query) + + # use simple whitespace tokenizing for core_text + text_tokens = core_text.split(" ") + + char_count = 0 + + for i, tok in enumerate(text_tokens): + + tok_clean = re.sub(r"[,.;:()?'-]", "", tok) + + for qt in query_tokens: + if qt == tok_clean.lower(): + matches_found.append([char_count, tok]) + break + + char_count += len(tok) + 1 + + return matches_found + + def highlighter(self,matches, core_string, highlight_start_token="", + highlight_end_token="", exclude_stop_words=True): + + """ Utility function to 'highlight' a selected token, based on matches, typically found + in locate_query_match function - useful for visual display of a matching keyword. """ + + # assumes by default: + # highlight_start_token = "" + # highlight_end_token = "" + # -- highlight can be any markup/html/css that will be inserted into the text for formatting + # around the highlighted word + + updated_string = "" + cursor_position = 0 + stop_word_list = [] + + if exclude_stop_words: + stop_word_list = self.get_stop_words_master_list() + + for mat in matches: + starter = mat[0] + keyword = mat[1] + + go_ahead = True + if exclude_stop_words: + if keyword in stop_word_list: + go_ahead = False + + if go_ahead: + + updated_string += core_string[cursor_position:starter] + updated_string += highlight_start_token + + # updated_string += keyword + # og_keyword preserves capitalization of original string + og_keyword = core_string[starter:(starter+len(keyword))] + updated_string += og_keyword + updated_string += highlight_end_token + + cursor_position = starter + len(keyword) + + if cursor_position < len(core_string): + updated_string += core_string[cursor_position:] + + return updated_string + + def package_answer(self, raw_query, text_core, answer_window, x): + + """ Takes a raw_query, text and answer_window as input and returns a context window around matches + to the query with the size of the answer_window. """ + + answer = [] + l = len(text_core) + + for t in range(0, l): + match = 0 + if text_core[t].lower() == raw_query[0].lower(): + if (t + len(raw_query)) < l: + for z in range(1, len(raw_query)): + + if text_core[t + z].lower() == raw_query[z].lower(): + match = z + else: + match = -1 + break + if match > 1: + + stop_slice = min(t + len(raw_query) + answer_window, t + l) + ans = text_core[t + len(raw_query) + 1:stop_slice] + doc = x['doc_ID'] + block = x['block_ID'] + page_num = x['master_index'] + fn = x['file_source'] + text_out = x['text'] + slice = t + len(raw_query) + 1 + answer.append((fn, doc, block, page_num, raw_query, slice, ans, text_out)) + + return answer + + def split_context_row (self, context_row): + + """ Splits a context row - internal utility method to support Graph class. """ + + entries_list = [] + entries_weights = [] + + for z in range(0,len(context_row)): + entries_list.append(context_row[z][0]) + entries_weights.append(int(context_row[z][1])) + + return entries_list, entries_weights + + def dataset_smart_packager(self, text_block, min_th=200, max_th=400): + + """ Deprecated - will remove in future release. """ + + # best outcome is to split at the end of a sentence + # use simple regex command to split the sentence on end punctuation (e.g., '.', '!', '?') + + sentences = list(re.split('(?<=[.!?])', text_block)) + + if len(sentences) == 1 or len(sentences) == 0: + # easy case - text block ends with "." -> return the whole block + return text_block, "" + + if len(sentences) > 1: + # check if last sentence ends with exclamation mark - otherwise, return as remainder + last_sentence = sentences[-1] + if last_sentence.endswith(".") or last_sentence.endswith("!") or last_sentence.endswith("?"): + return text_block, "" + else: + # re-assemble the sentences (excluding the last fragment) + output_text = "" + remainder_text = "" + for x in range(0, len(sentences) - 1): + if len(output_text) + len(sentences[x]) < max_th: + output_text += sentences[x] + " " + else: + remainder_text += sentences[x] + " " + + remainder_text += last_sentence + + if len(output_text) < min_th: + # in this case, retain the text_block as "remainder" and keep going + return "", text_block + else: + # the assembled sentences are longer than the min threshold + # if the remainder is very short, then append to output + if len(remainder_text) > 20: + return output_text, remainder_text + output_text += " " + remainder_text + return output_text, "" + + # something has gone wrong unexpectedly if this is reached + return text_block, "" + + def replace_word_numbers(self, evidence): + + """ Replaces word numbers with the actual number value. + + -- uses the word2number python library, which can be imported separately with pip install. + """ + + evidence_toks = evidence.split(" ") + + word_numbers_lookup = {"zero": 0, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5, "six": 6, + "seven": 7, "eight": 8, "nine": 9, "ten": 10, "eleven": 11, "twelve": 12, + "thirteen": 13, "fourteen": 14, "fifteen": 15, "sixteen": 16, "seventeen": 17, + "eighteen": 18, "nineteen": 19, "twenty": 20, "thirty": 30, "forty": 40, "fifty": 50, + "sixty": 60, "seventy": 70, "eighty": 80, "ninety": 90, "hundred": 100, + "thousand": 1000, "million": 1000000, "billion": 1000000000, "percent": 0.01} + + num_toks_in_progress = "" + text_with_numbers = "" + build_num = False + nums_in_text_list = [] + percent_flag = False + + token_index_of_match_found = [] + + for i, toks in enumerate(evidence_toks): + + if toks in word_numbers_lookup or (build_num and toks in ["and", "plus"]): + build_num = True + if toks not in ["and", "plus", "percent", "percentage"]: + num_toks_in_progress += toks + " " + if toks in ["percent", "percentage"]: + percent_flag = True + + else: + # add any number in progress, if any + if build_num: + + if percent_flag: + try: + from word2number import w2n + my_num = w2n.word_to_num(num_toks_in_progress) * 0.01 + except: + my_num = -9999.1234 + logger.info("update: could not import word2number to look for 'number-words' - if " + "you wish to use, `pip3 install word2number`") + else: + try: + from word2number import w2n + my_num = w2n.word_to_num(num_toks_in_progress) + except: + my_num = -9999.1234 + logger.info("update: could not import word2number to look for 'number-words' - if " + "you wish to use, `pip3 install word2number`") + + if my_num != -9999.1234: + text_with_numbers += str(my_num) + " " + nums_in_text_list.append(my_num) + + # new add - aug 26 + token_index_of_match_found.append(i) + + build_num = False + percent_flag = False + num_toks_in_progress = "" + + # add next token + text_with_numbers += toks + " " + + logger.info(f"update: text_with_numbers output: {text_with_numbers}") + logger.info(f"update: nums found list: {nums_in_text_list}") + + return text_with_numbers, nums_in_text_list, token_index_of_match_found + + def convert_media_file_to_wav(self, path_to_file_to_convert, save_path=None, file_out="converted_file.wav"): + + """ Utility method that converts wide range of video/audio file formats into .wav for transcription. + To use this method requires two separate installs: + + 1. pydub - e.g., `pip3 install pydub` + 2. lib install ffmpeg, e.g., brew install ffmpeg (MacOS) + """ + + # import ffmpeg -> need to import the core lib (brew install ffmpeg) + + try: + from pydub import AudioSegment + except: + raise DependencyNotInstalledException("pydub") + + # format + # format = "m4a" works + fmt = path_to_file_to_convert.split(".")[-1] + if fmt not in ["mp3", "m4a", "mp4", "wma", "aac", "ogg", "flv"]: + logger.warning(f"warning: file format - {fmt} - is not recognized and can not be converted.") + return None + + try: + given_audio = AudioSegment.from_file(path_to_file_to_convert, format=fmt, channels=2, frame_rate=16000) + outfile_path = os.path.join(save_path, file_out) + given_audio.export(outfile_path, format="wav") + except: + logger.warning(f"warning: could not successfully convert file @ {path_to_file_to_convert} to .wav - " + f"one common issue is the need to install ffmpeg which is a core audio/video " + f"processing library. It can be installed with apt (linux) ; brew (mac) ; or " + f"downloaded directly (windows).") + return None + + return outfile_path + + def secure_filename(self, fn): + + """ New utility method to remove os.sep from proposed filenames. """ + + # strip os.sep from file name + safe_file_name = str(fn) + if safe_file_name.startswith(os.sep): + safe_file_name = safe_file_name[1:] + + # removes os separator + secure_fn = safe_file_name.replace(os.sep, "_") + + # converts spaces into underscores + secure_fn = secure_fn.replace(" ", "_") + + return secure_fn + + def split_ocr_special_field1(self,special_field_text): + + """ Utility method to unpack a special_field text from an OCR block that will have the link + back to the original document and block id. """ + + doc_block = special_field_text.split("&") + output_dict = {} + + for elements in doc_block: + + key, value = elements.split("-") + try: + value = int(value) + except: + logger.warning(f"warning: could not convert value into integer as expected - {key} - {value}") + + output_dict.update({key: value}) + + return output_dict + + @staticmethod + def file_checksum(fp, fn, hash_type="sha256"): + + """ Creates File Checksum against a selected file with options to configure the hash_type, which must be + a hash supported by hashlib. If valid type not found, then automatic triage to 'sha256'. """ + + hash_output = None + + try: + import hashlib + + if hasattr(hashlib, hash_type): + hash_builder = getattr(hashlib, hash_type)() + else: + logging.warning(f"Utilities - file_checksum - selected hash type - {hash_type} - not supported -" + f"defaulting to sha256") + hash_builder = hashlib.sha256() + + # handle content in binary form + f = open(os.path.join(fp, fn), "rb") + + while chunk := f.read(4096): + hash_builder.update(chunk) + + hash_output = hash_builder.hexdigest() + + except: + logger.warning(f"Utilities - file_checksum - could not create file hash hex for: \n" + f"-- file: {fn}\n" + f"-- folder: {fp}\n" + f"-- hash type: {hash_type}") + + return hash_output + + @staticmethod + def create_hash_stamp (fp, save=True, hash_fn="hash_record", hash_type="sha256", + ignore_file_extensions=None,ignore_files=None, **kwargs): + + """ Creates Hash Stamp for all files in a folder. + + -- "hash_type" is 'sha256' by default, but can be configured to any hash type supported by hashlib + + -- If save is set to True (default), then writes as a JSON file into the folder using a filename that is a + concatenation of hash_fn and hash_type + + -- Will attempt to not over-write an existing hash record. If a matching filename is found, + then a fast triage will be applied to append a long random number to the file name - + note: it is unlikely but possible for a name space collision. Will enhance config and safety + options in future releases. + + """ + + import random + hash_record = {} + + # save as .json file and add hash_type by default at the end of the name + hash_full_name = hash_fn + "_" + hash_type + ".json" + + fp_files = os.listdir(fp) + + for file in fp_files: + + if file == hash_full_name: + + if save: + r = random.randint(0,10000000) + rec_core = str(hash_full_name).split(".")[0] + hash_full_name = rec_core + "_" + str(r) + ".json" + logging.warning(f"Utilities - create_hash_stamp - found existing hash_record with same name - " + f"attempting to create new hash record file with name - {hash_full_name}.") + + ignore = False + if ignore_file_extensions: + ft = file.split(".")[-1] + if ft.lower() in ignore_file_extensions or ft.upper() in ignore_file_extensions: + ignore = True + + if ignore_files: + if file in ignore_files: + ignore = True + + if not ignore: + hash_value = Utilities().file_checksum(fp, file, hash_type=hash_type) + hash_record.update({file: hash_value}) + + time_stamp = Utilities().get_current_time_now() + + hash_record.update({"time_stamp": time_stamp}) + + # option to add **kwargs to the stamp, e.g., user and related info + full_record = {**hash_record, **kwargs} + + if save: + + logger.debug(f"Utilities - create_hash_stamp - config output: {full_record}") + + import json + f = open(os.path.join(fp, hash_full_name), "w") + j = json.dumps(full_record, indent=1) + f.write(j) + f.close() + + return full_record + + @staticmethod + def compare_hash (fp, hash_fn="hash_record", hash_type="sha256", selected_files=None, ignore_pattern="hash", + ignore_file_extensions=None,ignore_files=None): + + """ Compares two hashes from a folder path (fp) - + + 1. An existing hash saved in the hash_fn file passed to the method. + 2. A new hash dynamically created against each file in the folder path. + + By default, the method will ignore files that start with "hash" but this can be disabled by setting + ignore_pattern to None or "" + + If only interested in hashes against a subset of the files, then an optional list of selected files + can be passed in the selected_files parameter - and only files matching those names will be + compared for hash consistency. + + """ + + import json + import os + + hash_full_name = hash_fn + "_" + hash_type + ".json" + + try: + hash_file = json.load(open(os.path.join(fp, hash_full_name), "r",errors='ignore',encoding='utf-8-sig')) + except: + logger.debug(f"Utilities - compare_hash_record - could not find an existing hash file at: " + f"{os.path.join(fp, hash_full_name)}. Will create new hash record, but will not " + f"be able to provide a meaningful comparison.") + hash_file = {} + + new_hash_record = Utilities().create_hash_stamp(fp, hash_fn=hash_fn, hash_type=hash_type, save=False, + ignore_file_extensions=ignore_file_extensions, + ignore_files=ignore_files) + + # apply any pruning of certain files + + if selected_files: + + # only compare files in the selected_files list + keys = list(new_hash_record.keys()) + + for key in keys: + if key not in selected_files: + del(new_hash_record[key]) + + else: + + # generally review all files with a few exclusions by default + keys = list(new_hash_record.keys()) + + # don't compare the hash of the time_stamp entry, which will be different + if "time_stamp" in new_hash_record: + del(new_hash_record["time_stamp"]) + + # ignore files starting with 'hash' by default + if ignore_pattern: + + for k in keys: + if k.startswith(ignore_pattern): + logger.debug(f"Utilities - compare_hash - ignoring - {k}") + del(new_hash_record[k]) + + hashed_item_count = len(new_hash_record.items()) + + matched_count = 0 + confirmed = {} + extra_keys = [] + values_changed = [] + confirmed_files = [] + + for key, value in new_hash_record.items(): + matched = False + if key in hash_file: + if value == hash_file[key]: + matched = True + matched_count += 1 + confirmed.update({key:value}) + confirmed_files.append(key) + else: + logger.debug(f"Utilities - compare_hash - value not matching for key - {key}") + values_changed.append(key) + else: + logger.debug(f"Utilities - compare_hash - extra key - {key} - in hash_file not found in original hash") + extra_keys.append(key) + + output_dict = {"hashed_file_count": hashed_item_count, + "validated_file_count": matched_count, + "extra_keys": extra_keys, + "changed_files": values_changed, + "validated_files": confirmed_files} + + return output_dict + + +class CorpTokenizer: + + """ Simple Custom 'Whole-word' Tokenizer implementation """ + + def __init__(self, lower_case=True, remove_punctuation=True, remove_stop_words=True, + remove_numbers=True, one_letter_removal=False): + + self.lower_case = lower_case + self.remove_punctuation = remove_punctuation + self.remove_stop_words = remove_stop_words + self.remove_numbers = remove_numbers + self.one_letter_removal = one_letter_removal + + def tokenize(self, text): + + """ Tokenizes an input text. """ + + # strip the whitespace from the beginning and end of the text so we can tokenize the data + text = text.strip() + + # start with basic whitespace tokenizing, + # this line will split on whitespace regardless of tab or multi-spaces between words + text2 = text.split() + + if self.remove_punctuation: + text2 = Utilities().clean_list(text2) + + if self.lower_case: + text_l = [] + for z in range(0, len(text2)): + text_l.append(str(text2[z]).lower()) + text2 = text_l + + if self.remove_stop_words: + text2 = Utilities().remove_stop_words(text2) + + if self.remove_numbers: + text_n = [] + for z in range(0, len(text2)): + if not str(text2[z]).isnumeric(): + text_n.append(text2[z]) + text2 = text_n + + if self.one_letter_removal: + text_out = [] + for z in range(0, len(text2)): + if len(text2[z]) > 1: + text_out.append(text2[z]) + text2 = text_out + + return text2 + + +class TextChunker: + + """ Text Chunker - input is a big chunk of text and output is a chunked set of smaller text chunks. """ + + # simple class that can be inserted for OCR, Text or HTML + # class expects to be passed a big chunk of text, e.g., output from OCR or full read of text file + # --will chop up blocks out of the text + # --uses a "chisel" approach, so starts with 'max_block_size' and looks back to find sentence edges + # --in testing with a number of files, it results in avg block size ~500 with 90%+ ending on sentence or \n\r + + def __init__(self, text_chunk=None, max_char_size=600, look_back_char_range=300): + + self.text_chunk = text_chunk + self.max_char_size = max_char_size + self.look_back_range = look_back_char_range + + self.chunks = [] + + self.avg_char_size = 0 + self.smallest_chunk = self.max_char_size + self.largest_chunk = 0 + self.chunks_ending_with_period = 0 + + def convert_text_to_chunks (self): + + """ Converts text into chunks. """ + + starter = 0 + + while starter < len(self.text_chunk): + + if (starter + self.max_char_size) < len(self.text_chunk): + stopper = starter + self.max_char_size + else: + stopper = len(self.text_chunk) + + smooth_stop = self.smooth_edge(starter, stopper) + chunk = self.text_chunk[starter:smooth_stop] + + starter = smooth_stop + + # if very short chunk, then concatenate with the previous chunk + if len(chunk) < self.look_back_range: + if len(self.chunks) > 0: + self.chunks[-1] += chunk + else: + self.chunks.append(chunk) + + else: + # general case - create next chunk + # chunk_pp = re.sub("[\n\r]", " ", chunk) + self.chunks.append(chunk) + + if len(chunk) < self.smallest_chunk: + self.smallest_chunk = len(chunk) + + if len(chunk) > self.largest_chunk: + self.largest_chunk = len(chunk) + + if len(chunk) > 0: + if ord(chunk[-1]) in [46,10,13]: + self.chunks_ending_with_period += 1 + + self.avg_char_size += len(chunk) + + return self.chunks + + def smooth_edge(self,starter,stopper): + + """ Produces a 'smooth edge' between starter and stopper. """ + + # default case is to return the whole text sample as single chunk + smooth_stop = stopper + + # look back is the full range that will be reviewed to find proper stopping point + if (stopper - self.look_back_range) > starter: + look_back = stopper - self.look_back_range + else: + look_back = starter + + # best case - look for a period + found_period = -1 + for x in range(stopper-1,look_back,-1): + + # found a period followed by white space marker (space, \n, \r) - best case + if ord(self.text_chunk[x]) == 46: + + # first confirm that '.' is followed by white space or is the end of the text + if x+1 == stopper or ord(self.text_chunk[x + 1]) in [32, 13, 10]: + + # exclude 'several edge cases where '.' is not a reliable sentence end + short_window = self.text_chunk + if x > 5: + short_window = self.text_chunk[x-5:x-1] + + # (A) first edge case - "two periods close to each other", e.g., "x.y." + if "." not in short_window and short_window != "": + + # (B) second edge case - "period after number in list", e.g., "point 2." + if not 47 < ord(short_window[-1]) < 58: + + # (C) third edge case - common abbreviations + if short_window[:-2] != "Mr" and short_window[:3] != "Mrs" and short_window[:2] != "Dr": + + # if none of (A) - (B) - (C) or apply, then consider period valid stopping point + found_period = x + 1 + break + + # alternate solid stopper is presence of \n\n | \n\r | \r\r -> usually marks a section/para end + if ord(self.text_chunk[x]) in [10,13]: + if x+1 == stopper or ord(self.text_chunk[x+1]) in [10,13]: + found_period = x+1 + break + + # if found a period, then smooth stop is the char right after the period + if found_period > - 1: + smooth_stop = found_period + + else: + # if no period found, then next best case is to look for whitespace between words + for y in range(stopper - 1, look_back,-1): + + # look for a white space separator + if ord(self.text_chunk[y]) in [32, 13, 10]: + smooth_stop = y + break + + # if no period or white space found, then return the original stopper + + return smooth_stop + + +class AgentWriter: + + """ Specialized Logging utility designed for capturing 'agent' and 'agent-like' inference outputs where + the intent is to capture a 'show-your-work' chain of logic, rather than a traditional log output, which is + generated through logging. AgentWriter provides three basic options for capturing + this output: + + -- 'screen' - default - writes to stdout + -- 'file' - writes to file + -- 'off' - turns off (no action taken) + """ + + def __init__(self, mode=None): + + # options configured through global LLMWareConfigs + if mode: + self.mode = mode + else: + self.mode = LLMWareConfig().get_agent_writer_mode() + + self.fp_base = LLMWareConfig().get_llmware_path() + self.fn = LLMWareConfig().get_agent_log_file() + + self.file = os.path.join(self.fp_base, self.fn) + + if self.mode == "screen": + self.writer = sys.stdout + self.file = None + elif self.mode == "file": + if os.path.exists(self.file): + self.writer = open(self.file, "a") + else: + self.writer = open(self.file, "w") + else: + # takes no action + self.writer = None + self.file = None + + def write(self, text_message): + + """ Writes output to selected output stream. """ + + if self.writer: + if self.mode == "file": + try: + escape_ansi_color_codes = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + text_message = escape_ansi_color_codes.sub('', text_message) + except: + pass + self.writer.write(text_message+"\n") + + def close(self): + + """ Closes at end of process if needed to close the file. """ + + if self.file: + self.writer.close() + + +class LocalTokenizer: + + """ LocalTokenizer class manages and caches tokenizer.json files for common base models used in + LLMWare. Enables re-instantiating the Tokenizer directly using the standalone tokenizers library, + regardless of the model class, e.g., very useful for GGUF and post-processing prompt analysis. """ + + def __init__(self, tokenizer_fn=None, tokenizer_name=None): + + # tokenizer files kept in llmware repo @ llmware/bonchon for easy access + self.hf_repo_tokenizers = "llmware/bonchon" + + # map of "tokenizer_name" to "tokenizer_fn" + self.tokenizer_mapping = {"phi3": "tokenizer_phi3.json", + "tiny_llama": "tokenizer_tl.json", + "stablelm": "tokenizer_stablelm.json", + "yi": "tokenizer_yi.json", + "qwen": "tokenizer_qw.json", + "mistral": "tokenizer_mistral.json", + "llama2": "tokenizer_ll2.json", + "llama3": "tokenizer_ll3.json", + "bert": "tokenizer_bert.json", + "roberta": "tokenizer_roberta.json", + "xlm_roberta": "tokenizer_roberta_xlm.json", + "phi2": "tokenizer_phi2.json", + "gpt2": "tokenizer_gpt2.json" + } + + # keeping a few key parameters hard-coded for easy access and assignment + self.supported_model = { + + # phi-3 tokenizer + "tokenizer_phi3.json": {"bos_id": [1], "bos_token": "", + "eos_id": [32000,32001,32007], "eos_token": "<|endoftext|>"}, + + # phi-2 tokenizer + "tokenizer_phi2.json": {"bos_id": [50256], "bos_token": "<|endoftext|>", + "eos_id": [50256], "eos_token": "<|endoftext|>"}, + + # stablelm-3b tokenizer + "tokenizer_stablelm.json": {"bos_id": [0], "bos_token": "<|endoftext|>", + "eos_id": [0], "eos_token": "<|endoftext|>"}, + + # tiny llama tokenizer + "tokenizer_tl.json": {"bos_id": [1], "bos_token": "", "eos_id": [2], "eos_token": ""}, + + # 01-ai yi tokenizer + "tokenizer_yi.json": {"bos_id": [1], "bos_token": "<|startoftext|>", + "eos_id": [2], "eos_token": "<|im_end|>"}, + + # Qwen tokenizer + "tokenizer_qw.json": {"bos_id": [151643], "bos_token": "<|endoftext|>", + "eos_id": [151645], "eos_token": "<|im_end|>"}, + + # Mistral tokenizer + "tokenizer_mistral.json": {"bos_id": [1], "bos_token": "", "eos_id": [2], "eos_token": ""}, + + # llama2 tokenizer + "tokenizer_ll2.json": {"bos_id": [1], "bos_token": "", "eod_id": [2], "eos_token": ""}, + + # llama3 tokenizer + "tokenizer_ll3.json": {"bos_id": [128000], "bos_token": "<|begin_of_text|>", + "eos_id": [128001], "eos_token": "<|end_of_text|>"}, + + # bert tokenizer + "tokenizer_bert.json": {"pad_id": [0]}, + + # roberta tokenizer + "tokenizer_roberta.json": {"bos_id": [0], "bos_token": "", "eos_id": [2], "eos_token": "", + "pad_id": [1], "pad_token": ""}, + + # roberta xlm tokenizer + "tokenizer_roberta_xlm.json": {"bos_id": [0], "bos_token": "", "eos_id": [2], "eos_token": "", + "pad_id": [1], "pad_token": ""}, + + # gpt2 tokenizer + "tokenizer_gpt2.json": {"bos_id": [50256], "bos_token": "", "eos_id": [50256], "eos_token": ""}, + + # granite tokenizer + "tokenizer_granite.json": {"bos_id": 100257, "bos_token": "<|end_of_text|>", + "eos_id": [100257], "eos_token": "<|end_of_text|>"}, + + "tokenizer_phi4.json": {"bos_id": 100257, "bos_token": "<|endoftext|>", + "eos_id": [100257, 100265], "eos_token": "<|endoftext|>"}, + + "tokenizer_phi4_mini.json": {"bos_id": 199999, "bos_token": "<|endoftext|>", + "eos_id": [199999, 200020], "eos_token": "<|endoftext|>"}, + + "tokenizer_stablelm_1_6.json": {"bos_id": 100257, "bos_token": "<|endoftext|>", + "eos_id": [100257], "eos_token": "<|endoftext|>"}, + + "tokenizer_gemma.json": {"bos_id": 2, "bos_token": "", + "eos_id": [1], "eos_token": ""}, + + "tokenizer_mistral_chat.json": {"bos_id": 1, "bos_token": "", + "eos_id": [2, 32000, 32768], "eos_token": ["", "<|im_end|>"]}, + + } + + self.tokenizer_name = tokenizer_name + self.tokenizer_fn = tokenizer_fn + self.tokenizer = None + + # default dummy values + self.bos_id = [-1] + self.bos_token = "" + self.eos_id = [-1] + self.eos_token = "" + self.pad_id = [-1] + self.pad_token = "" + + if tokenizer_name: + if tokenizer_name in self.tokenizer_mapping: + self.tokenizer_fn = self.tokenizer_mapping[tokenizer_name] + + if self.tokenizer_fn: + + if self.tokenizer_fn in self.supported_model: + for keys in self.supported_model[self.tokenizer_fn]: + setattr(self, keys, self.supported_model[self.tokenizer_fn][keys]) + + # will attempt to load the tokenizer + self.load_tokenizer(self.tokenizer_fn) + + else: + raise LLMWareException(f"LocalTokenizer - could not identify selected tokenizer - " + f"tokenizer file - {self.tokenizer_fn} - " + f"tokenizer name - {self.tokenizer_name}") + + def load_tokenizer(self, tokenizer_fn=None): + + if tokenizer_fn: + self.tokenizer_fn = tokenizer_fn + + try: + # use the tokenizer library to instantiate - less overhead than transformers library when + # only the tokenizer is needed + from tokenizers import Tokenizer + except: + raise LLMWareException(message="LocalTokenizer class requires tokenizers to be installed, e.g., " + "`pip3 install tokenizers`.") + + model_repo_path = LLMWareConfig().get_model_repo_path() + + if not os.path.exists(model_repo_path): + os.mkdir(model_repo_path) + + tokenizers_cache = os.path.join(model_repo_path, "tokenizers_local_cache") + + if not os.path.exists(tokenizers_cache): + os.mkdir(tokenizers_cache) + + tokenizers_in_cache = os.listdir(tokenizers_cache) + + logger.debug(f"LocalTokenizer - tokenizers found in cache: {tokenizers_in_cache}") + + if tokenizer_fn not in tokenizers_in_cache: + logger.info(f"LocalTokenizer - need to fetch tokenizer - {tokenizer_fn}") + self.fetch_tokenizer_from_hb(self.hf_repo_tokenizers, tokenizer_fn, tokenizers_cache) + + self.tokenizer = Tokenizer.from_file(os.path.join(tokenizers_cache, tokenizer_fn)) + + return True + + def fetch_tokenizer_from_hb(self, repo, file, local_path): + + """ Retrieves the tokenizer json file from the llmware/bonchon repo. """ + + # need to pull from HF cache + from huggingface_hub import hf_hub_download + + downloader = hf_hub_download(repo, file, local_dir=local_path, local_dir_use_symlinks=False) + + # remove ongoing links, if any, created by attributes not in the file repo + files_created = os.listdir(local_path) + if ".huggingface" in files_created: + try: + shutil.rmtree(os.path.join(local_path,".huggingface")) + logger.debug("LocalTokenizers cache: removed .huggingface") + except: + logger.info(f"LocalTokenizers cache: .huggingface folder created in repo and not auto-removed.") + pass + + if ".gitattributes" in files_created: + try: + os.remove(os.path.join(local_path, ".gitattributes")) + logger.debug("LocalTokenizers cache - removed: .gitattributes") + except: + logger.info(f"LocalTokenizers cache - .gitattributes created in repo and not auto-removed.") + pass + + if ".cache" in files_created: + try: + shutil.rmtree(os.path.join(local_path, ".cache")) + logger.debug("LocalTokenizers cache - removed: .cache") + except: + logger.info(f"LocalTokenizers cache - .cache folder created in repo and not auto-removed.") + pass + + return True + + def encode(self, seq): + + """ Encode the sequence and return the token ids in a list. """ + + return self.tokenizer.encode(seq, add_special_tokens=False).ids + + def decode(self, seq, strip_bos_token=True): + + """ Decode a list of tokens and return the decoded string. """ + + if not isinstance(seq, list): + seq = [seq] + + decoded = self.tokenizer.decode(seq, skip_special_tokens=False) + + if strip_bos_token: + if decoded.startswith(self.bos_token): + decoded = decoded[len(self.bos_token):] + + return decoded + + +class Sources: + + """Implements a source batching designed to build a set of 'source materials' for a source_client_obj, which + is passed into the constructor for Sources. + + Sources is responsible for providing a consistent set of metadata attributes and algorithm for 'chunking' a large + input source into multiple separate context prompts (string) to send to a LLM, while preserving of all of the + metadata from the original source, to be able to post-processing comparison with individual chunks, e.g., + preserving the page number. + + The class is intended to support a wide range of potential 'source clients' with the only requirement that + the source client has a 'source_materials' attribute, which will be written to as part of constructing + the source batches. + + Other optional attributes of a source_client will be checked and used if available: + -- tokenizer + -- context_window_size + -- batch_separator + + Parameters + ---------- + source_client_obj : object + Designed for Prompt or Agent client objects, but can be any Python object with a "source_materials" attribute + + tokenizer: Optional - pass a tokenizer directly + + context_window_size: Optional - default of 1000 as the target context size (this can be made larger, and is + set conservatively to better support accuracy with smaller models + + batch_separator: string used to aggregate distinct entries to build a larger prompt (e.g., "\n" by default) + + """ + + def __init__(self, source_client_obj, tokenizer=None,context_window_size=1000,batch_separator="\n"): + + self.source_client= source_client_obj + self.tokenizer= tokenizer + self.context_window_size=context_window_size + self.batch_separator=batch_separator + + self.source_input_keys = ["text", "file_source", "page_num"] + self.source_output_keys = [] + + self.source_keys = ["batch_id", "text", "metadata", "biblio", "batch_stats", "batch_stats.tokens", + "batch_stats.chars", "batch_stats.samples"] + + self.source_metadata = ["batch_source_num", "evidence_start_char", "evidence_stop_char", + "source_name", "page_num", "doc_id", "block_id"] + + if not tokenizer: + resolved_tokenizer = self.resolve_tokenizer() + + if not resolved_tokenizer: + logger.debug(f"Sources - could not resolve tokenizer to use - may lead to downstream source " + f"packaging issues.") + + if hasattr(self.source_client, "context_window_size"): + self.context_window_size = self.source_client.context_window_size + + if hasattr(self.source_client, "batch_separator"): + self.batch_separator = self.source_client.batch_separator + + if not hasattr(source_client_obj, "source_materials"): + raise LLMWareException(message=f"Sources - expects a source_client object with a 'source_materials' " + f"attribute - which by default can be set to an empty list, e.g., []") + + def resolve_tokenizer(self): + + """ Will attempt to resolve the tokenizer associated with the Prompt, and use a default tokenizer + as a fallback if not found in the Prompt object. """ + + found_tokenizer = False + + # option 1 - pull the tokenizer from the prompt directly + if hasattr(self.source_client, "tokenizer"): + if self.source_client.tokenizer: + self.tokenizer = self.source_client.tokenizer + return True + + # option 2 - pull the 'tokenizer_local' file from the model card and instantiate + if not found_tokenizer: + if hasattr(self.source_client, "llm_model_card"): + if isinstance(self.source_client.llm_model_card, dict): + if "tokenizer_local" in self.source_client.llm_model_card: + tokenizer_fn = self.source_client.llm_model_card["tokenizer_local"] + try: + self.tokenizer = LocalTokenizer(tokenizer_fn=tokenizer_fn) + return True + except: + pass + + # option 3 - fallback + if not found_tokenizer: + # use llama2 tokenizer as a default fallback + # note: the tokenizer is used primarily for 'counting' against the context window, so if the + # wrong tokenizer is used, the counts may be off, and the batch sizes not perfectly optimized + # relative to the context window, but there should be any other detrimental impacts + + default_tokenizer = "tokenizer_ll2.json" + try: + self.tokenizer = LocalTokenizer(tokenizer_fn=default_tokenizer) + except: + logger.warning("Could not resolve tokenizer - some functionality may not work correctly." + "\nHave you installed tokenizers, e.g., `pip3 install tokenizers`") + return True + + return False + + def token_counter(self, text_sample): + + """ Token counter utility """ + + if not self.tokenizer: + self.resolve_tokenizer() + + if self.tokenizer: + # toks = self.tokenizer.encode(text_sample).ids + toks = self.tokenizer.encode(text_sample) + else: + toks = "" + logger.warning(f"Sources - could not identify a tokenizer - batch size allocation compared to " + f"context window may not be possible.") + + return len(toks) + + def tokenize (self, text_sample): + + """ Tokenize utility """ + + if not self.tokenizer: + self.resolve_tokenizer() + + # toks = self.tokenizer.encode(text_sample).ids + toks = self.tokenizer.encode(text_sample) + return toks + + def package_source(self, retrieval_material, aggregate_source=True, add_to_prompt=True, + backup_source_filename="user_provided_unknown_source"): + + """ Generalized source packager + --assumes minimal metadata - doc_name, page_num and text chunk + --add to existing 'state' source & create new batch on top if overflow """ + + # tracking variables + tokens_per_batch = [] + samples_per_batch = [] + sample_counter = 0 + doc_sources = {} + + doc_sources_per_batch = {} + + biblio_per_batch = [] + batches = [] + meta = [] + + samples = [] + + for i, q in enumerate(retrieval_material): + + # simple deduplication check to remove identical entries - more 'cleaning' options can be offered over time + if q not in samples: + samples.append(q) + + # default + current_batch = "" + token_counter = 0 + batch_metadata = [] + batch_id = 0 + char_counter = 0 + + if aggregate_source: + # start current batch with the last entry in source materials and aggregate from this point + if len(self.source_client.source_materials) > 0: + + # pull up the last 'in-progress' entry in current source materials state + current_batch = self.source_client.source_materials[-1]["text"] + token_counter = self.token_counter(current_batch) + char_counter = len(current_batch) + batch_metadata = self.source_client.source_materials[-1]["metadata"] + batch_stats = self.source_client.source_materials[-1]["batch_stats"] + batch_id = len(self.source_client.source_materials) - 1 + + # experiment + doc_sources_per_batch = self.source_client.source_materials[-1]["biblio"] + + # end - experiment + + # 'pop' the last entry 'in-progress' off the list + self.source_client.source_materials = self.source_client.source_materials[:-1] + + samples_chunked = [] + + for x in range(0,len(samples)): + + t = self.token_counter(samples[x]["text"]) + + if t > self.context_window_size: + chunks = self.chunk_large_sample(samples[x]) + samples_chunked += chunks + else: + samples_chunked.append(samples[x]) + + samples = samples_chunked + + for x in range(0, len(samples)): + + t = self.token_counter(samples[x]["text"]) + + if "file_source" in samples[x]: + source_fn = samples[x]["file_source"] + else: + source_fn = backup_source_filename + + if "page_num" in samples[x]: + page_num = samples[x]["page_num"] + else: + if "master_index" in samples[x]: + page_num = samples[x]["master_index"] + else: + # if can not retrieve from metadata, then set as default - page 1 + page_num = 1 + + if "doc_id" in samples[x]: + doc_id = samples[x]["doc_id"] + else: + # if can not retrieve from metadata, then set as default - doc_id 1 + doc_id = 1 + + if "block_id" in samples[x]: + block_id = samples[x]["block_id"] + else: + # if can not retrieve from metadata, then set as default - block_id 1 + block_id = 1 + + # keep aggregating text batch up to the size of the target context_window for selected model + if (t + token_counter) < self.context_window_size: + + # appends separator at end of sample text before adding the next chunk of text + current_batch += samples[x]["text"] + self.batch_separator + batch_char_len = len(current_batch) + + new_source = {"batch_source_id": len(batch_metadata), + "evidence_start_char": char_counter, + # remove adding char_counter to evidence_stop_char + "evidence_stop_char": batch_char_len, + "source_name": source_fn, + "page_num": page_num, + "doc_id": doc_id, + "block_id": block_id, + } + + batch_metadata.append(new_source) + + char_counter = batch_char_len + token_counter += t + + # new trackers + sample_counter += 1 + if source_fn not in doc_sources: + doc_sources.update({source_fn: [page_num]}) + else: + if page_num not in doc_sources[source_fn]: + doc_sources[source_fn].append(page_num) + + if source_fn not in doc_sources_per_batch: + doc_sources_per_batch.update({source_fn: [page_num]}) + else: + if page_num not in doc_sources_per_batch[source_fn]: + doc_sources_per_batch[source_fn].append(page_num) + + else: + # capture number of tokens in batch + tokens_per_batch.append(token_counter) + samples_per_batch.append(sample_counter) + sample_counter = 1 + + biblio_per_batch.append(doc_sources_per_batch) + + # doc_sources_per_batch = {} + + if "file_source" in samples[x]: + doc_filename = samples[x]["file_source"] + else: + doc_filename = backup_source_filename + + if "page_num" in samples[x]: + page_num = samples[x]["page_num"] + else: + # adding check for master_index + if "master_index" in samples[x]: + page_num = samples[x]["master_index"] + else: + # if no page_num identified, then default is page 1 + page_num = 1 + + # doc_sources_per_batch.update({doc_filename: [page_num]}) + biblio = doc_sources_per_batch + + # reset + doc_sources_per_batch = {} + + batches.append(current_batch) + meta.append(batch_metadata) + + if add_to_prompt: + # corrected batch_id counter + new_batch_dict = {"batch_id": batch_id, "text": current_batch, "metadata": batch_metadata, + "biblio": biblio, "batch_stats": + {"tokens": token_counter, + "chars": len(current_batch), + "samples": len(batch_metadata)}} + + self.source_client.source_materials.append(new_batch_dict) + + batch_id += 1 + + # reset current_batch -> current snippet + current_batch = samples[x]["text"] + token_counter = t + new_source = {"batch_source_id": 0, + "evidence_start_char": 0, + "evidence_stop_char": len(samples[x]["text"]), + "source_name": source_fn, + "page_num": page_num, + "doc_id": doc_id, + "block_id": block_id, + } + + batch_metadata = [new_source] + char_counter = len(samples[x]["text"]) + + # insert change - dec 23 + if doc_filename not in doc_sources_per_batch: + doc_sources_per_batch.update({doc_filename: [page_num]}) + else: + if page_num not in doc_sources_per_batch[doc_filename]: + doc_sources_per_batch[doc_filename].append(page_num) + # end - insert change + + if len(current_batch) > 0: + + batches.append(current_batch) + meta.append(batch_metadata) + + if add_to_prompt: + # change batch_id from batches -> len(batches) + new_batch_dict = {"batch_id": batch_id, "text": current_batch, "metadata": batch_metadata, + "biblio": doc_sources_per_batch, "batch_stats": {"tokens": token_counter, + "chars": len(current_batch), + "samples": len(batch_metadata)}} + + self.source_client.source_materials.append(new_batch_dict) + + # batch_id += 1 + + # add new stats for last batch + tokens_per_batch.append(token_counter) + samples_per_batch.append(sample_counter) + biblio_per_batch.append(doc_sources_per_batch) + + new_sources = {"text_batch": batches, "metadata_batch": meta, "batches_count": len(batches)} + + return new_sources + + def chunk_large_sample(self, sample): + + """ If single sample bigger than the context window, then break up into smaller chunks """ + + chunks = [] + max_size = self.context_window_size + sample_len = self.token_counter(sample["text"]) + + chunk_count = sample_len // max_size + if max_size * chunk_count < sample_len: + chunk_count += 1 + + stopper = 0 + base_dict = {} + for key, values in sample.items(): + base_dict.update({key:values}) + + sample_tokens = self.tokenize(sample["text"]) + + for x in range(0,chunk_count): + starter = stopper + stopper = min((x+1)*max_size,sample_len) + new_chunk_tokens = sample_tokens[starter:stopper] + new_dict = base_dict + new_dict.update({"text":self.tokenizer.decode(new_chunk_tokens)}) + chunks.append(new_dict) + + return chunks + diff --git a/llmware/web_services.py b/llmware/web_services.py new file mode 100644 index 0000000..7816095 --- /dev/null +++ b/llmware/web_services.py @@ -0,0 +1,1394 @@ + +# Copyright 2023-2026 llmware + +# Licensed under the Apache License, Version 2.0 (the "License"); you +# may not use this file except in compliance with the License. You +# may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. See the License for the specific language governing +# permissions and limitations under the License. + + +""" The web_services module implements classes to enable integrated access to popular web services within +LLMWare pipelines. """ + + +import logging +import os +import shutil + +from llmware.configs import LLMWareConfig, LLMWareException + +logger = logging.getLogger(__name__) + + +class WikiKnowledgeBase: + + """ WikiKnowledgeBase implements Wikipedia API """ + + def __init__(self): + + # importing here to suppress log warnings produced by urllib3 + import urllib3 + urllib3.disable_warnings() + + self.user_agent = "Examples/3.0" + + try: + from wikipediaapi import Wikipedia, ExtractFormat + except ImportError: + raise LLMWareException(message="Exception: pip install `wikipediaapi` required.") + + self.wiki = Wikipedia(user_agent=self.user_agent, extract_format=ExtractFormat.WIKI, verify=False) + self.wiki_search_api_url = 'http://en.wikipedia.org/w/api.php' + + def get_article(self, article_name): + + """ Retrieves a Wikipedia article by name. """ + + article_response = {"title": "", "summary": "", "text": ""} + + try: + page_py = self.wiki.page(article_name) + + if page_py.exists(): + + logger.info(f"update: page_py - {page_py.title} - {page_py.summary}") + logger.info(f"update: text - {page_py.text}") + + article_response = {"title": page_py.title, "summary": page_py.summary, "text": page_py.text} + + else: + logger.info(f"update: connected with Wikipedia - selected article does not exist - " + f"{article_name}") + + except: + logger.error(f"error: could not retrieve wikipedia article - please try again") + + return article_response + + def search_wikipedia(self, query, result_count=10, suggestion=False): + + """ Searches Wikipedia database API with a search 'topic' query. """ + + # output result + output = [] + + # search params passed to the wikipedia api + search_params = {'list': 'search', 'srprop': '', 'srlimit': result_count, 'srsearch': query, + 'format': 'json', 'action': 'query'} + + if suggestion: search_params['srinfo'] = 'suggestion' + + headers = {'User-Agent': self.user_agent} + + try: + import requests + r = requests.get(self.wiki_search_api_url, params=search_params, headers=headers, verify=False) + + for i, title in enumerate(r.json()["query"]["search"]): + + logger.info(f"update: wiki results - {i} - {title}") + + new_entry = {"num": i, "title": title["title"], "pageid": title["pageid"]} + output.append(new_entry) + + except: + logger.error("error: could not connect with Wikipedia to retrieve search results") + + return output + + +class YFinance: + + """ YFinance class implements the Yahoo Finance API. """ + + def __init__(self, ticker=None): + + """ + Widely used Yahoo Finance API - key object = " + TickerObj = yahooFinance.Ticker("META") + print("All Info : ", TickerObj.info) + for keys, values in TickerObj.info.items(): + print("keys: ", keys, values) + + # display Company Sector + print("Company Sector : ", TickerObj.info['sector']) + + # display Price Earnings Ratio + print("Price Earnings Ratio : ", TickerObj.info['trailingPE']) + + # display Company Beta + print(" Company Beta : ", TickerObj.info['beta']) + print(" Financials : ", TickerObj.get_financials()) + """ + + self.company_info = None + + self.financial_summary_keys = ["shortName", "symbol","marketCap", "totalRevenue", "ebitda", "revenueGrowth", "grossMargins", + "freeCashflow", "priceToSalesTrailing12Months", "grossMargins","currency"] + + self.stock_summary_keys = ["shortName", "symbol", "exchange","bid", "ask", "fiftyTwoWeekLow", "fiftyTwoWeekHigh", "symbol", + "shortName", "longName", "currentPrice", "targetHighPrice", "targetLowPrice", + "returnOnAssets", "returnOnEquity", "trailingPE", "forwardPE", "volume", + "forwardEps", "pegRatio", "currency"] + + self.risk_summary_keys = ["shortName","symbol", "auditRisk", "boardRisk", "compensationRisk", "shareHolderRightsRisk", "overallRisk", + "shortName", "longBusinessSummary"] + + self.company_summary_keys = ["shortName", "longName", "symbol", "marketCap", "companyOfficers", "website", + "industry", "sector", "longBusinessSummary", "fullTimeEmployees"] + + self.keys = ["address1", "city", "state", "zip", "country", "phone","website","industry", + "industryDisp", "sector", "sectorDisp", "longBusinessSummary", "fullTimeEmployees", + "companyOfficers", "auditRisk", "boardRisk", "compensationRisk", "shareHolderRightsRisk", + "overallRisk", "previousClose", "open", "dayLow", "dayHigh", "regularMarketPreviousClose", + "regularMarketOpen", "regularMarketDayLow", "regularMarketDayHigh", "payoutRatio", "beta", + "trailingPE", "forwardPE", "volume", "regularMarketVolume", "averageVolume", + "averageVolume10days", "bid", "ask", "bidSize", "askSize", "marketCap", "fiftyTwoWeekLow", + "fiftyTwoWeekHigh", "priceToSalesTrailing12Months", "fiftyDayAverage", "twoHundredDayAverage", + "trailingAnnualDividendRate", "trailingAnnualDividendYield", "currency", "enterpriseValue", + "profitMargins", "floatShares", "sharesOutstanding", "sharesShort", "sharesShortPriorMonth", + "sharesShortPreviousMonthDate", "dateShortInterest", "sharesPercentSharesOut", + "heldPercentInsiders", "heldPercentInstitutions", "shortRatio", "shortPercentOfFloat", + "impliedSharesOutstanding", "bookValue", "priceToBook", "lastFiscalYearEnd", + "nextFiscalYearEnd", "mostRecentQuarter", "earningsPerQuarterlyGrowth", "netIncomeToCommon", + "trailingEps", "forwardEps", "pegRatio", "enterpriseToRevenue", "enterpriseToEbitda", + "52WeekChange", "SandP52WeekChange", "exchange", "quoteType", "symbol", "underlyingSymbol", + "shortName", "longName", "currentPrice", "targetHighPrice", "targetLowPrice", "targetMeanPrice", + "targetMedianPrice", "recommendationMean", "recommendationKey", "numberOfAnalystOpinions", + "totalCash", "totalCashPerShare", "ebitda", "totalDebt", "quickRatio", "currentRatio", + "totalRevenue", "debtToEquity", "revenuePerShare", "returnOnAssets" "returnOnEquity", "grossProfits", + "freeCashflow", "operatingCashflow", "earningsGrowth", "revenueGrowth", "grossMargins", + "ebitdaMargins", "operatingMargins", "financialCurrency", "trailingPegRatio"] + + try: + import yfinance + except ImportError or ModuleNotFoundError: + raise LLMWareException(message="Exception: YFinance library not installed - " + "fix with `pip3 install yfinance`") + + self.ticker = None + + if ticker: + + self.ticker = self._prep_ticker(ticker) + + try: + self.company_info = yfinance.Ticker(self.ticker) + except: + logger.warning(f"YFinance - attempted to retrieve company info based on ticker lookup with " + f"ticker - {self.ticker} - and did not succeed. Please check the ticker.") + + else: + self.company_info = None + + def _prep_ticker(self, ticker): + + """ Yfinance API is particular about the ticker format, so a little prep handling to + maximize likelihood of positive response. """ + + # if ticker includes exchange (often used in formal formats of exchange), then strip + ticker_core = ticker.split(":")[-1] + + # check that all characters are alpha + ticker_remediated = "" + for letters in ticker_core: + if 97 <= ord(letters) <= 122: + cap_letter = chr(ord(letters)-32) + ticker_remediated += cap_letter + elif 65 <= ord(letters) <= 90: + ticker_remediated += letters + elif 48 <= ord(letters) <= 57: + ticker_remediated += letters + else: + # skip and do not include + logging.warning(f"YFinance - prep ticker - found unexpected letter in ticker - removing - {letters}") + + # TODO: add more remediation steps + + return ticker_core + + def ticker(self, company_ticker, **kwargs): + + """ Retrieves company information based on the company_ticker. """ + + self.ticker = self._prep_ticker(company_ticker) + + try: + import yfinance + except ImportError: + raise LLMWareException(message="Exception: need to `pip install yfinance` library.") + + try: + company_info = yfinance.Ticker(self.ticker) + except: + company_info = {} + logger.warning(f"YFinance - ticker - not successful looking up company information using the " + f"company ticker - {self.ticker}") + + return company_info + + def get_company_summary(self, ticker=None, **kwargs): + + """ Retrieves company summary based on the ticker. """ + + try: + import yfinance + except ImportError: + raise LLMWareException(message="Exception: need to `pip install yfinance` library.") + + self.ticker = self._prep_ticker(ticker) + + output_info = {} + + try: + company_info = yfinance.Ticker(self.ticker).info + except: + company_info = {} + logger.warning(f"YFinance - ticker - not successful looking up company summary using the " + f"ticker - {ticker}") + + for targets in self.company_summary_keys: + found_key = False + for keys, values in company_info.items(): + if targets == keys: + output_info.update({targets: values}) + found_key = True + if not found_key: + output_info.update({targets: "NA"}) + logger.warning(f"YFinance - get_company_summary - could not find {targets} in web service response.") + + return output_info + + def get_financial_summary(self, ticker=None, **kwargs): + + """ Retrieves financial summary based on the ticker. """ + + try: + import yfinance + except ImportError: + raise LLMWareException(message="Exception: need to `pip install yfinance` library.") + + if ticker: + self.ticker = self._prep_ticker(ticker) + + try: + company_info = yfinance.Ticker(self.ticker).info + except: + company_info = {} + logger.warning(f"YFinance - ticker - not successful looking up company summary using the " + f"ticker - {self.ticker}") + + output_info = {} + for targets in self.financial_summary_keys: + found_key = False + for keys, values in company_info.items(): + if targets == keys: + output_info.update({targets: values}) + found_key = True + if not found_key: + output_info.update({targets:"NA"}) + logger.warning(f"YFinance - get_financial_summary - could not find {targets} in web service response.") + + return output_info + + def get_stock_summary(self, ticker=None, **kwargs): + + """ Retrieves the stock summary based on ticker. """ + + try: + import yfinance + except ImportError: + raise LLMWareException(message="Exception: need to `pip install yfinance` library.") + + if ticker: + if isinstance(ticker, dict): + ticker = ticker["ticker"] + + self.ticker = self._prep_ticker(ticker) + + output_info = {} + try: + company_info = yfinance.Ticker(self.ticker).info + except: + company_info = {} + logger.warning(f"YFinance - ticker - not successful looking up company summary using the " + f"ticker - {self.ticker}") + + for targets in self.stock_summary_keys: + key_found = False + for keys,values in company_info.items(): + if targets == keys: + output_info.update({targets: values}) + key_found = True + if not key_found: + output_info.update({targets:"NA"}) + logger.warning(f"YFinance - get_stock_summary - could not find {targets} in web service response.") + + return output_info + + +class WebSiteParser: + + """ WebSiteParser implements a website-scraping parser. It can be accessed directly, or through the Parser + and Library classes indirectly. + + Scraping web content is generally permissible in most cases, although ethical guidelines and sensible best + practices should be followed. Scraped content does not grant any license to the content, and still must be + used in compliance with any associated copyrights. + + For a good introduction to recommended web scraping practices, see + https://monashdatafluency.github.io/python-web-scraping/section-5-legal-and-ethical-considerations/ + + This website parser implementation is designed to quickly extract content from HTML-oriented websites, and + is not intended for use on large commercial websites, which tend to be primarily dynamic javascript. + + If your local SSL certificate is out-of-date, you will likely receive errors - this can be updated easily + with a python command script, or you can select unverified_context=True to ignore + + """ + + def __init__(self, url_or_fp, link="/", save_images=True, reset_img_folder=False, local_file_path=None, + from_file=False, text_only=False, unverified_context=False): + + try: + from bs4 import BeautifulSoup + import requests + from urllib.request import urlopen, Request + import lxml + + except ModuleNotFoundError or ImportError: + raise LLMWareException(message="Exception: to use WebSiteParser requires additional Python " + "dependencies via pip install: " + "\n -- pip3 install beautifulsoup4 (or bs4)" + "\n -- pip3 install lxml" + "\n -- pip3 install requests" + "\n -- pip3 install urllib3") + + # note: for webscraping, unverified ssl are a common error + # to debug, if the risk environment is relatively low, set `unverified_context` = True, although + # preferred method is to update the ssl certificate to remove this error + self.unverified_context=unverified_context + + # by default, assume that url_or_fp is a url path + self.url_main = url_or_fp + + # by default, will get images and links + self.text_only = text_only + + # by passing link - provides option for recursive calls to website for internal links + if link == "/": + self.url_link = "" + else: + self.url_link = link + + self.url_base = self.url_main + self.url_link + + # check for llmware path & create if not already set up + if not os.path.exists(LLMWareConfig.get_llmware_path()): + # if not explicitly set up by user, then create folder directory structure + LLMWareConfig.setup_llmware_workspace() + + if not local_file_path: + # need to update this path + self.local_dir = os.path.join(LLMWareConfig.get_llmware_path(), "process_website/") + else: + self.local_dir = local_file_path + + if reset_img_folder: + if os.path.exists(self.local_dir): + # important step to remove & clean out any old artifacts in the /tmp/ directory + shutil.rmtree(self.local_dir) + os.makedirs(self.local_dir, exist_ok=True) + + if not os.path.exists(self.local_dir): + os.makedirs(self.local_dir, exist_ok=True) + + if from_file: + # interpret url as file_path and file_name + try: + html = open(url_or_fp, encoding='utf-8-sig', errors='ignore').read() + bs = BeautifulSoup(html, features="lxml") + self.html = bs.findAll() + success_code = 1 + self.text_only = True + except: + logger.error(f"error: WebSite parser- could not find html file to parse at {url_or_fp}") + success_code = -1 + self.text_only = True + else: + # this is the most likely default case -interpret url_or_fp as url + try: + req = Request(self.url_base, headers={'User-Agent': 'Mozilla/5.0'},unverifiable=True) + + if self.unverified_context: + import ssl + ssl._create_default_https_context = ssl._create_unverified_context + html = urlopen(req).read() + else: + html = urlopen(req).read() + + bs = BeautifulSoup(html, features="lxml") + + self.bs = bs + self.html = bs.findAll() + + out_str = "" + for x in self.html: + out_str += str(x) + " " + + with open(os.path.join(self.local_dir, "my_website.html"), "w", encoding='utf-8') as f: + f.write(out_str) + f.close() + + success_code = 1 + + except Exception as e: + success_code = -1 + raise LLMWareException(message=f"Exception: website_parser could not connect to website - " + f"caught error - {e}. Common issues: \n" + f"1. Update your certificates in the Python path, e.g., " + f"'Install Certificates.command'\n" + f"2. Set unverified_context=True in the constructor for " + f"WebSiteParser.\n" + f"Note: it is also possible that the website does not exist, " + f"or that it is restricted and rejecting your request for any " + f"number of reasons. The website may have other restrictions on " + f"programmatic 'bot' access, and if so, those should be followed.") + + self.save_images = save_images + self.image_counter = 0 + self.links = [] + self.mc = None + self.entries = None + self.core_index = [] + self.header_text = [] + self.internal_links = [] + self.external_links = [] + self.other_links = [] + + # meta-data expected in library add process + self.source = str(self.url_base) + self.success_code = success_code + + def website_main_processor(self, img_start, output_index=True): + + """ Main processing of HTML scraped content and converting into blocks. """ + + logger.info(f"update: WebSite Parser - initiating parse of website: {self.url_main}") + + output = [] + counter = 0 + # by passing img_start explicitly- enables recursive calls to links/children sites + img_counter = img_start + + long_running_string = "" + + # new all_text to remove duplications + all_text = [] + + internal_links = [] + external_links = [] + header_text = [] + unique_text_list = [] + unique_header_list = [] + + last_text = "" + last_header = "" + + text = "" + + for elements in self.html: + + content_found = 0 + img = "" + img_success = 0 + img_url = "" + img_name = "" + + link = "" + link_type = "" + + # text = "" + text_dupe_flag = False + entry_type = "text" + + # if text only, then skip checks for images and links + if not self.text_only: + + if "property" in elements.attrs: + if elements.attrs["property"] == "og:image": + if "content" in elements.attrs: + + img_extension = elements["content"] + img_success, img, img_url, img_name = \ + self.image_handler(img_extension, elements, img_counter) + + if img_success == 1: + img_counter += 1 + content_found += 1 + + if "src" in elements.attrs: + + img_extension = elements["src"] + img_success, img, img_url, img_name = self.image_handler(img_extension, elements, img_counter) + + if img_success == 1: + img_counter += 1 + content_found += 1 + + if "href" in elements.attrs: + + if elements.attrs["href"]: + link_success, link, link_type = self.link_handler(elements) + content_found += 1 + + if link_success == 0: + # skip .js files and other formatting in link crawling + # link_success == 0 if not .js // ==1 if .js file + + if link_type == "internal": + if link != "/": + if link not in internal_links: + internal_links.append(link) + + if link_type == "external": + external_links.append(link) + + # main check for text + if elements.get_text(): + + get_text = 1 + + if elements.attrs == {}: + get_text = -1 + + if "type" in elements.attrs: + # skip css and javascript + if elements.attrs["type"] in ["text/css", "text/javascript", "application/ld+json", + "application/jd+json"]: + + get_text = -1 + + # wip - generally associated with javascript inline script + if "charset" in elements.attrs: + get_text = -1 + + if "jsname" in elements.attrs: + get_text = -1 + + if "jscontroller" in elements.attrs: + get_text = -1 + + if get_text == 1: + + # text handler + s_out = "" + + # alt for consideration to clean up string + # s_out += string.replace('\n', ' ').replace('\r', ' ').replace('\xa0', ' ').replace('\t', ' ') + + js_markers = ["{", "function", "(function", "if (", "window", "on", "#", "this.", ".", + "var", ";", "html", "@"] + + keep_adding = True + + for string in elements.stripped_strings: + + if not s_out: + + # kludge - list of exclusions to remove the most common javascript in site + for marker in js_markers: + if string.startswith(marker): + keep_adding = False + break + + # if found at start of any substring - high probability inline js - break + if string.startswith("(function") or string.startswith("window.") or string.startswith("this."): + break + + if keep_adding: + s_out += string + " " + else: + break + + text += s_out + + if text.strip(): + header_entry = [] + + if text not in unique_text_list: + unique_text_list.append(text) + content_found += 1 + long_running_string += text + " " + last_text = text + text_dupe_flag = False + else: + text_dupe_flag = True + + if "h1" in elements.name: + header_entry = (counter, "h1", text) + + if "h2" in elements.name: + header_entry = (counter, "h2", text) + + if "h3" in elements.name: + header_entry = (counter, "h3", text) + + if header_entry: + if text not in unique_header_list: + last_header = text + header_text.append(header_entry) + unique_header_list.append(text) + + # if looking for images and links, then prioritize in attribution + if not self.text_only: + if img and img_success == 1: + entry_type = "image" + else: + if link: + entry_type = "link" + else: + if text: + entry_type = "text" + else: + content_found = 0 + else: + entry_type = "text" + + if content_found > 0: + master_index = (self.url_main, self.url_link, counter) + + if text and not text_dupe_flag: + + entry = {"content_type": entry_type, + "text": text, + "image": {"image_name": img_name, "image_url": img_url}, + "link": {"link_type": link_type, "link": link}, + "master_index": master_index, + "last_header": last_header} + + counter += 1 + # save entry if image, or if (A) text > 50 and (B) not a dupe + if entry_type == "image" or (len(text) > 50 and text not in all_text): + output.append(entry) + all_text.append(text) + text = "" + + self.image_counter = img_counter + self.internal_links = internal_links + self.external_links = external_links + self.header_text = header_text + + if header_text: + header_text_sorted = sorted(header_text, key=lambda x: x[1]) + self.header_text = header_text_sorted + + self.core_index = output + self.entries = len(output) + + logger.info(f"update: WebSite Parser - completed parsing: {self.url_main}") + logger.info(f"update: extracted {len(self.core_index)} elements") + + if self.image_counter > 0: + logger.info(f"update: extracted {self.image_counter} images and saved @ path: {self.local_dir}") + + if not output_index: + return len(output), img_counter + + return self.core_index + + def link_handler(self, elements): + + """ Handles processing of links found in main page content. """ + + link_out = "" + link_type = "" + js_skip = 0 + + if elements.attrs["href"].endswith(".js"): + link_out = elements.attrs["href"] + link_type = "js" + js_skip = 1 + + if elements.attrs["href"].endswith(".ico") or elements.attrs["href"].endswith(".ttf"): + link_out = elements.attrs["href"] + link_type = "other_formatting" + js_skip = 1 + + if elements.attrs["href"].endswith(".css"): + link_out = elements.attrs["href"] + link_type = "css" + js_skip = 1 + + if elements.attrs["href"].startswith(self.url_base): + # save relative link only + link_out = elements.attrs["href"][len(self.url_base):] + link_type = "internal" + + if str(elements.attrs["href"])[0] == "/": + # relative link + if elements.attrs["href"]: + if not elements.attrs["href"].startswith("//"): + link_out = elements.attrs["href"] + link_type = "internal" + + if elements.attrs["href"].startswith("https://") and \ + not elements.attrs["href"].startswith(self.url_base): + # website but not the url_base - external link + link_out = elements.attrs["href"] + link_type = "external" + + return js_skip, link_out, link_type + + def image_handler(self, img_extension, elements, img_counter): + + """ Handles and processes images found in main content. """ + + success = -1 + img_raw = [] + image_name = "" + full_url = "" + + try: + img_raw, response_code, full_url = self._request_image(img_extension, elements) + + if response_code == 200: + + if self.save_images: + + # need to capture img type, e.g., .jpg + img_type = "" + if img_extension.endswith("png"): img_type = "png" + if img_extension.endswith("jpg") or img_extension.endswith("jpeg"): img_type = "jpg" + if img_extension.endswith("tiff"): img_type = "tiff" + if img_extension.endswith("svg"): img_type = "svg" + + # secondary check if not at end - break off at '?' query string + if img_type == "": + original_img_name = img_extension.split("/")[-1] + original_img_name = original_img_name.split("?")[0] + if original_img_name.endswith("png"): img_type = "png" + if original_img_name.endswith("jpg") or img_extension.endswith("jpeg"): img_type = "jpg" + if original_img_name.endswith("tiff"): img_type = "tiff" + if original_img_name.endswith("svg"): img_type = "svg" + + # only save image if valid img format found + if img_type in ("png", "jpg", "svg", "tiff"): + image_name = "image{}.{}".format(img_counter, img_type) + fp = self.local_dir + image_name + s = self._save_image(img_raw, fp) + success = 1 + + else: + logger.info(f"update: WebSite - found image OK but could not " + f"figure out img type: {img_extension} ") + + except: + logger.info(f"warning: WebSite - could not retrieve potential image: {elements.attrs['src']}") + success = -1 + + return success, img_raw, full_url, image_name + + def _save_image(self, img_raw, fp): + + """ Internal utility to save images found. """ + + with open(fp, 'wb') as f: + img_raw.decode_content = True + shutil.copyfileobj(img_raw, f) + + return 0 + + def _save_image_website(self, fp, img_num, doc_id, save_file_path): + + """ Internal utility for images. """ + + # internal method to save image files and track counters + + img_type = img_num.split(".")[-1] + img_core = img_num[len("image"):].split(".")[0] + + # image name of format: image{{doc_ID}}_{{img_num}}.png + new_img_name = "image" + str(doc_id) + "_" + str(img_core) + "." + img_type + created = 0 + + img = open(os.path.join(fp, img_num), "rb").read() + if img: + f = open(os.path.join(save_file_path, new_img_name), "wb") + f.write(img) + f.close() + created += 1 + + return new_img_name, created + + def _request_image(self, img_extension, img): + + """ Retrieve images from links. """ + try: + import requests + except ImportError: + raise LLMWareException(message="Exception: could not import `requests` library which is a required " + "dependency for web parsing.") + + # relative link - refers back to main index page + # check if url_main gives better performance than .url_base + + url_base = self.url_main + url_ext = img_extension + + full_url = url_ext + + if url_ext: + if url_ext.startswith("https:"): + # this is an external link - just use the source + full_url = url_ext + + if url_ext.startswith("/"): + # relative ID - add url_base to get img + + full_url = url_base + url_ext + + r = requests.get(full_url, stream=True, headers={'User-Agent': 'Mozilla/5.0'}) + + return r.raw, r.status_code, full_url + + def get_all_links(self): + + """ Utility to retrieve all links. """ + + # note: not called by the main handler - kept as direct callable method + + internal_links = [] + external_links = [] + other_links = [] + js_links = [] + + for content in self.html: + + found = 0 + js = 0 + + if "href" in content.attrs: + if content.attrs["href"]: + + if content.attrs["href"].endswith(".js"): + js_links.append(content.attrs["href"]) + js = 1 + + if content.attrs["href"].startswith(self.url_base): + # save relative link only + out = content.attrs["href"][len(self.url_base):] + internal_links.append(out) + found = 1 + + if str(content.attrs["href"])[0] == "/": + # relative link + out = content.attrs["href"] + if out: + # skip double // + if not out.startswith("//"): + internal_links.append(out) + found = 1 + + if content.attrs["href"].startswith("https://") and \ + not content.attrs["href"].startswith(self.url_base): + # website but not the url_base - external link + out = content.attrs["href"] + external_links.append(out) + found = 1 + + if found == 0: + other_links.append(content.attrs["href"]) + + self.internal_links = internal_links + self.external_links = external_links + self.other_links = other_links + + top_links = [] + + for z in range(0, len(internal_links)): + + link_tokens = internal_links[z].split("/") + for y in range(0, len(self.mc)): + if self.mc[y][0].lower() in link_tokens: + if internal_links[z] not in top_links: + top_links.append(internal_links[z]) + break + + link_results = {"internal_links": internal_links, "external_links": external_links, + "other_links": other_links, "top_links": top_links} + + return link_results + + def get_all_img(self, save_dir): + + """ Utility to get all images from html pages. """ + + # note: not called by main handler - kept as separate standalone method + + counter = 0 + for content in self.html: + counter += 1 + if "src" in content.attrs: + if str(content).startswith("/llmware.git + ``` +3. **Create a Branch**: Create a new branch for your topic: + ```bash + git checkout -b my-topic-branch + ``` +4. **Run Tests**: Navigate to the `tests/` folder and run: + ```bash + ./run-tests.py -s + ``` + Ensure there are no failures. +5. **Commit Changes**: Make your changes and commit them to your branch. +6. **Push to GitHub**: Push your branch to your GitHub fork: + ```bash + git push origin my-topic-branch + ``` +7. **Submit a Pull Request**: Open a [pull request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) for review. + +**Synchronize Your Fork**: Before submitting your PR, ensure your fork is up-to-date to minimize merge conflicts: +```bash +git remote add upstream git@github.com:llmware-ai/llmware.git +git fetch upstream +git checkout upstream/main -b my-topic-branch +``` + +### Questions or Ideas? + +If you have questions or just want to brainstorm ideas, feel free to engage in our [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). We’re here to help! diff --git a/repo_docs/README_AWS.md b/repo_docs/README_AWS.md new file mode 100644 index 0000000..9c8158e --- /dev/null +++ b/repo_docs/README_AWS.md @@ -0,0 +1,134 @@ + + +Getting Started: `llmware` AMI from the AWS Marketplace +=============== + +The llmware python package is pre-installed on the AMI. Examples of how to use llmware can be found locally in ~/llmware/examples and can be run using "python3 [filename].py". + +The llmware package is updated frequenty, so it is strongly recommended to update the version running on your AMI with the following commands. To install the latest llmware package from PyPi: +``` + pip install llmware --upgrade +``` + +To pull the latest code from the llmware repository and gain access to the latest examples: + + ```bash + cd ~/llmware + git pull + + ``` +Some examples require LLM API Keys. You can edit the example code to include your own, or +update ```~/set-env.sh``` and run the following to get the environment variables loaded in your shell: + +``` +source ~/set-env.sh +``` + +If you have deployed on a NVIDIA GPU instance type (g5.x2large or g5.4xlarge are good and cost-effective options), then you'll need to run this additional script to update the AMI with the NVIDIA drivers and CUDA: + +``` +source <(curl https://raw.githubusercontent.com/llmware-ai/llmware/main/scripts/aws/update-ami-nvidia-gpu.sh) +``` + +_Note: You can expect a one-time delay of 60-90 seconds when running llmware on this instance for the first time. +This is due to how EBS storage volumes get loaded on-demand from the AMI storage snapshot._ + +MongoDB & Milvus +================ + +By default, MongoDB and Milvus are running on the AMI. + +To stop (or start) MongoDB or Milvus: + ```bash + sudo systemctl [stop|start] [mongodb|milvus] +``` + +To disable (or re-enable) MongoDB or Milvus starting on instance reboot: + ```bash +sudo systemctl [disable|enable] [mongodb|milvus] +``` + + +Using llmware +============== + +### 🔥 Start coding - Quick Start For RAG 🔥 +```python +# This example demonstrates Retrieval Augmented Retrieval (RAG) using the llmware package: +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.prompts import Prompt +from llmware.setup import Setup + +# Update this value with your own API Key, either by setting the env var or editing it directly here: +openai_api_key = os.environ["OPENAI_API_KEY"] + +# A self-contained end-to-end example of RAG +def end_to_end_rag(): + + # Create a library called "Agreements", and load it with llmware sample files + print (f"\n > Creating library 'Agreements'...") + library = Library().create_new_library("Agreements") + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"Agreements")) + + # Create vector embeddings for the library using the "industry-bert-contracts model and store them in Milvus + print (f"\n > Generating vector embeddings using embedding model: 'industry-bert-contracts'...") + library.install_new_embedding(embedding_model_name="industry-bert-contracts", vector_db="milvus") + + # Perform a semantic search against our library. This will gather evidence to be used in the LLM prompt + print (f"\n > Performing a semantic query...") + os.environ["TOKENIZERS_PARALLELISM"] = "false" # Avoid a HuggingFace tokenizer warning + query_results = Query(library).semantic_query("Termination", result_count=20) + + # Create a new prompter using the GPT-4 and add the query_results captured above + prompt_text = "Summarize the termination provisions" + print (f"\n > Prompting LLM with '{prompt_text}'") + prompter = Prompt().load_model("gpt-4", api_key=openai_api_key) + sources = prompter.add_source_query_results(query_results) + + # Prompt the LLM with the sources and a query string + responses = prompter.prompt_with_source(prompt_text, prompt_name="summarize_with_bullets") + for response in responses: + print ("\n > LLM response\n" + response["llm_response"]) + + # Finally, generate a CSV report that can be shared + print (f"\n > Generating CSV report...") + report_data = prompter.send_to_human_for_review() + print ("File: " + report_data["report_fp"] + "\n") + +end_to_end_rag() +``` +#### Response from end-to-end RAG example + +``` +> python examples/rag_with_openai.py + + > Creating library 'Agreements'... + + > Generating vector embeddings using embedding model: 'industry-bert-contracts'... + + > Performing a semantic query... + + > Prompting LLM with 'Summarize the termination provisions' + + > LLM response +- Employment period ends on the first occurrence of either the 6th anniversary of the effective date or a company sale. +- Early termination possible as outlined in sections 3.1 through 3.4. +- Employer can terminate executive's employment under section 3.1 anytime without cause, with at least 30 days' prior written notice. +- If notice is given, the executive is allowed to seek other employment during the notice period. + + > Generating CSV report... +File: /Users/llmware/llmware_data/prompt_history/interaction_report_Fri Sep 29 12:07:42 2023.csv +``` + +**Additional examples of how to use llmware can be found locally in ~/llmware/examples and can be run using `python3 .py`** + + +Need help or have questions? +============================ + +Check out the [llmware videos](https://www.youtube.com/@llmware) and [GitHub repository](https://github.com/llmware-ai/llmware). + +Reach out to us on [GitHub Discussions](https://github.com/llmware-ai/llmware/discussions). diff --git a/repo_docs/SECURITY.md b/repo_docs/SECURITY.md new file mode 100644 index 0000000..68b151d --- /dev/null +++ b/repo_docs/SECURITY.md @@ -0,0 +1,22 @@ +# Security Policy + +## Using llmware securely + +Due to the open connectivity and wide potential use cases of llmware, it is possible to use llmware in conjunction with a wide range of potential models and data sources. + +**Models** - be careful with models that you use, especially if the model is unknown to you, or demonstrates any unusual behavior. + +**Data Sources** - Be careful with data sources that you bring into llmware, both for any malicious content that may be in the sources, as well as potential data privacy or copyright concerns. + +If you believe that you have identified a security issue in any of these 3rd party components, we would recommend that you contact the original source of that component. In such cases, there is usually little that llmware can do to directly help, and we do not view such cases as security vulnerabilities of llmware, per se. + + +## Reporting a Vulnerability + +If you believe you have found a security vulnerability in llmware, or in a dependency of llmware, we encourage you to let us know right away. We will investigate all reports and do our best to quickly fix the problem. + +Please report security issues by sending an email to `security@aibloks.com`. + +Do __not__ submit an issue ticket or pull request or otherwise publicly disclose the issue. + +After receiving your email, we will assess respond as soon as possible and share what we plan to do. diff --git a/scripts/aws/launch-llmware-ami.sh b/scripts/aws/launch-llmware-ami.sh new file mode 100755 index 0000000..19d4a31 --- /dev/null +++ b/scripts/aws/launch-llmware-ami.sh @@ -0,0 +1,31 @@ +#! /bin/bash + +# This is an example of programmatically launching the llmware AMI from the 'aws' command line +# Details: +# instance-type: +# - t2.2xlarge is the default (non-GPU) instance type +# - g5.2xlarge or g5.4xlarge are good, cost-effective choices for Nvidia GPU +# key-name: +# - Update the command below to include your key pair name +# subnet-id: +# - Update the command below to include the target subnet-id +# block-device-mappings +# - The default storage size is 100GB, which is sufficient for modest workloads +# - If you plan to experiment with many models we recommend increasing that size below +# tag-specifications: +# - By default the command below will name the instance "llmware-001". You can change this below + +INSTANCE_TYPE=t2.2xlarge +KEY_NAME=YOUR-KEY-NAME +SUBNET_ID=YOUR-SUBNET-ID +STORAGE_SIZE=100 +INSTANCE_NAME=llmware-001 + +aws ec2 run-instances --image-id ami-0eef3676fbf4809e5 \ + --count 1 \ + --instance-type $INSTANCE_TYPE \ + --key-name $KEY_NAME \ + --subnet-id $SUBNET_ID \ + --block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=$STORAGE_SIZE}" \ + --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$INSTANCE_NAME}]" + diff --git a/scripts/aws/update-ami-nvidia-gpu.sh b/scripts/aws/update-ami-nvidia-gpu.sh new file mode 100755 index 0000000..ef30361 --- /dev/null +++ b/scripts/aws/update-ami-nvidia-gpu.sh @@ -0,0 +1,20 @@ +#! /bin/bash + +# This script updates the llmware 0.1.3 AMI with Cuda and required dependencies to take advantage of Nvidia GPU instances + +# Setup cuda keyring so that we just have to run a single 'apt update' below +wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb +sudo dpkg -i cuda-keyring_1.1-1_all.deb +rm cuda-keyring_1.1-1_all.deb + +# Install ubuntu + nvidia drivers +sudo apt update +sudo apt install -y ubuntu-drivers-common nvidia-driver-535 + +# Install cuda +sudo apt install -y cuda + +# Update .bashrc and the current shell's environment +echo "export PATH=/usr/local/cuda/bin:${PATH}" >> ~/.bashrc +echo "export LD_LIBRARY_PATH=/usr/local/cuda-12.2/lib64:${LD_LIBRARY_PATH}" >> ~/.bashrc +source ~/.bashrc diff --git a/scripts/devcontainer/README b/scripts/devcontainer/README new file mode 100644 index 0000000..b4b2e23 --- /dev/null +++ b/scripts/devcontainer/README @@ -0,0 +1,4 @@ +If you wish to use devcontainers for development in vscode. You will need to rename or create a symlink of this directory to .devcontainer and reload your window. This will trigger the an option to open the code in a container. Once the code is opened in a container you will be able to contribute like normally without having to install all the dependencies on your local system. Also, the development container provides access to your local home directory via the /code directory. + +quick how-to: +run this command on linux from the root llmware: ln -s devcontainer .devcontainer diff --git a/scripts/devcontainer/devcontainer.json b/scripts/devcontainer/devcontainer.json new file mode 100644 index 0000000..269a512 --- /dev/null +++ b/scripts/devcontainer/devcontainer.json @@ -0,0 +1,49 @@ +{ + + "name": "LLMWARE Dev", + + // This is the image to use if doing development on only the llmware code base + //"image": "provocoai/llmware:dev-01", / + // using the dockerComposeFile option allows us to setup + // the full env around the llm code base include milvus and mongodb. + "dockerComposeFile": "docker-compose.yaml", + + "service": "llmware", // we define the container to connect too + + // update based on individual perference and mount point defined below + "workspaceFolder": "/llmware", + + // "remoteUser": "llmware", //used with docker instead of podman + // "containerUser": "llmware", // used with docker instead of podman + + // when we close the devcontainer env make sure we clean up after ourselfs + "shutdownAction": "stopCompose", + + // needed for podman to keep the id's the same + "runArgs": [ + "--userns=keep-id" + ], + + // Define the mountpoints into the llmware container. + // I use $HOMEDIR/code/provoco as my default codebase and + // then set the target to /code so I have access to all my code in /code + "mounts" : [ + "source=${localEnv:HOME}/code/provoco,target=/code,Z,type=bind,consistency=cached" +// "source=${localENV:HOME}/.vscode-server,target=/root/.vscode-server,Z,type=bind,consistency=cached" + ], + + // setup the github cli feature so we can commit within the UI + "features": { + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + + "customizations": { + "vscode": { + "extensions": [ + "esbenp.prettier-vscode", // prettify the code extension + "ms-python.python", //python code extensions + "ms-python.vscode-pylance" // vscode python extension + ] + } + } +} diff --git a/scripts/devcontainer/devcontainer.json-single.bk b/scripts/devcontainer/devcontainer.json-single.bk new file mode 100644 index 0000000..748378d --- /dev/null +++ b/scripts/devcontainer/devcontainer.json-single.bk @@ -0,0 +1,53 @@ +{ + "name": "LLMWARE Dev", + //"build": { "dockerfile": "../Dockerfile" }, + "containerUser": "llmware", + "updateRemoteUserUID": true, + "image": "provocoai/llmware:dev-01", + "remoteUser": "llmware", + "runArgs": [ + "--userns=keep-id:uid=1000,gid=1000" + ], + + + // "runArgs": [ + // "--name", + // "${localWorkspaceFolderBasename}", // Container name + // "-it", + // "-l", + // "com.docker.compose.project=devcontainers" // Container group name + // ], + // you can setup your local directory in the devcontainer here. The mount line is an example and mounts your home directory into the /code directory + "mounts" : [ + "source=/home/harrison/code/provoco,target=/code,type=bind,consistency=cached,Z", + "source=/home/harrison/.vscode,target=/root/.vscode,type=bind,consistency=cached,Z" + ], + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + //"mongodb.mongodb-vscode", + "ms-python.python" + //"ms-azuretools.vscode-docker" + ] + } + }, + "postCreateCommand": "apt-get install ", + "features": {} + // "features": { + // "ghcr.io/devcontainers/features/docker-outside-of-docker:1": { + // "dockerDashComposeVersion": "v2" + // }, + // "ghcr.io/devcontainers/features/github-cli:1": {} + // }, + // "customizations": { + // "vscode": { + // "extensions": [ + // "esbenp.prettier-vscode", // prettify the code extension + // "ms-python.python", //python code extensions + // "ms-python.vscode-pylance" // vscode python extension + // ] + // } + // } +} diff --git a/scripts/devcontainer/docker-compose.yaml b/scripts/devcontainer/docker-compose.yaml new file mode 100644 index 0000000..f7809ae --- /dev/null +++ b/scripts/devcontainer/docker-compose.yaml @@ -0,0 +1,108 @@ +version: "3.5" + +services: + llmware: + container_name: llmware + image: provocoai/llmware:dev-01 + depends_on: + - "milvus" + - "mongodb" + command: sleep infinity + network_mode: host + + mongodb: + container_name: mongodb + image: mongo:5.0.10 + # To secure MongoDB, uncomment and set the following values + # environment: + # - MONGO_INITDB_DATABASE=admin + # - MONGO_INITDB_ROOT_USERNAME=admin + # - MONGO_INITDB_ROOT_PASSWORD=changeme + volumes: + - llmware-mongodb:/data/db:Z + ports: + - '27017:27017' + + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - llmware-etcd:/etcd:Z + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - llmware-minio:/minio_data:Z + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + milvus: + container_name: milvus + image: milvusdb/milvus:v2.3.0 + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - llmware-milvus:/var/lib/milvus:Z + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + depends_on: + - "etcd" + - "minio" + + + dev-neo4j: + container_name: devneo4j + hostname: neo4j + image: neo4j:5.15.0-community + ports: + - 7474:7474 + - 7687:7687 + restart: always + + volumes: + - $HOME/neo4j/data:/data:Z + - $HOME/neo4j/logs:/logs:Z + + environment: + - NEO4J_AUTH=none + +volumes: + llmware-mongodb: + driver: local + llmware-etcd: + driver: local + llmware-minio: + driver: local + llmware-milvus: + driver: local diff --git a/scripts/docker/Dockerfile b/scripts/docker/Dockerfile new file mode 100644 index 0000000..4922498 --- /dev/null +++ b/scripts/docker/Dockerfile @@ -0,0 +1,39 @@ +# Base image with Python 3.11 +FROM python:3.11-bookworm + +# Define build arguments +ARG USERNAME=llmware +ARG USER_UID=1000 +ARG USER_GID=${USER_UID} + +# Set environment variables +ENV PYTHONPATH=/llmware + +# Update and install required packages, optimize by combining the steps +RUN apt-get update && apt-get install -y \ + git \ + bash \ + postgresql \ + musl-dev \ + build-essential \ + libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Clone the repository and install Python package dependencies +RUN git clone https://github.com/llmware-ai/llmware.git /llmware \ + && cd /llmware/llmware \ + && pip install --no-cache-dir -r requirements.txt + +# Create a user with specified UID and GID, assign ownership of /llmware +RUN groupadd --gid ${USER_GID} ${USERNAME} \ + && useradd --uid ${USER_UID} --gid ${USER_GID} -m ${USERNAME} \ + && chown -R ${USERNAME}:${USER_GID} /llmware + +# Switch to the created user +USER ${USERNAME} + +# Set the working directory +WORKDIR /llmware + +# Set the default command to run bash +CMD ["/bin/bash"] diff --git a/scripts/docker/docker-compose-neo4j.yaml b/scripts/docker/docker-compose-neo4j.yaml new file mode 100644 index 0000000..8a130bb --- /dev/null +++ b/scripts/docker/docker-compose-neo4j.yaml @@ -0,0 +1,23 @@ +version: '3.5' + + +services: + + dev-neo4j: + container_name: devneo4j + hostname: neo4j + image: neo4j:5.15.0-community + + ports: + - 7474:7474 + - 7687:7687 + + restart: always + + volumes: + - $HOME/neo4j/data:/data + - $HOME/neo4j/logs:/logs + + environment: + - NEO4J_AUTH=none + diff --git a/scripts/docker/docker-compose-pgvector.yaml b/scripts/docker/docker-compose-pgvector.yaml new file mode 100644 index 0000000..76eaa9a --- /dev/null +++ b/scripts/docker/docker-compose-pgvector.yaml @@ -0,0 +1,18 @@ + +version: '3.5' + +services: + db: + hostname: db + image: ankane/pgvector + ports: + - 5432:5432 + restart: always + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD="" + - POSTGRES_HOST_AUTH_METHOD=trust + volumes: + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + diff --git a/scripts/docker/docker-compose-qdrant.yaml b/scripts/docker/docker-compose-qdrant.yaml new file mode 100644 index 0000000..9cc001d --- /dev/null +++ b/scripts/docker/docker-compose-qdrant.yaml @@ -0,0 +1,42 @@ +services: + mongodb: + container_name: mongodb + image: mongo:5.0.10 + restart: always + # To secure MongoDB, uncomment and set the following values + # environment: + # - MONGO_INITDB_DATABASE=admin + # - MONGO_INITDB_ROOT_USERNAME=admin + # - MONGO_INITDB_ROOT_PASSWORD=changeme + volumes: + - llmware-mongodb:/data/db + ports: + - '27017:27017' + + qdrant: + image: qdrant/qdrant:latest + restart: always + container_name: qdrant + ports: + - 6333:6333 + - 6334:6334 + expose: + - 6333 + - 6334 + - 6335 + configs: + - source: qdrant_config + target: /qdrant/config/production.yaml + volumes: + - qdrant_data:/qdrant_data + +volumes: + llmware-mongodb: + driver: local + qdrant_data: + driver: local + +configs: + qdrant_config: + content: | + log_level: INFO \ No newline at end of file diff --git a/scripts/docker/docker-compose-redis-stack.yaml b/scripts/docker/docker-compose-redis-stack.yaml new file mode 100644 index 0000000..a8f39cc --- /dev/null +++ b/scripts/docker/docker-compose-redis-stack.yaml @@ -0,0 +1,20 @@ + + +version: '3.5' + +services: + + redis: + image: redis/redis-stack-server:latest + container_name: redis-standalone + ports: + - 6379:6379 + restart: always + environment: + - "REDIS_ARGS=--enable-debug-command yes --enable-module-command yes" + volumes: + - redis_data:/data + +volumes: + redis_data: + driver: local diff --git a/scripts/docker/docker-compose.yaml b/scripts/docker/docker-compose.yaml new file mode 100644 index 0000000..32fef45 --- /dev/null +++ b/scripts/docker/docker-compose.yaml @@ -0,0 +1,114 @@ +version: "3.5" + +services: + llmware: + container_name: llmware + image: provocoai/llmware:dev-01 + volumes: + - $HOME/code/provoco:/code:Z + - $HOME/.vscode:/root/.vscode:Z + network_mode: service:mongodb + # - service:milvus + # - service:etcd + # - service:minio + # - service:devneo4j + command: sleep infinity + + mongodb: + container_name: mongodb + image: mongo:5.0.10 + # To secure MongoDB, uncomment and set the following values + # environment: + # - MONGO_INITDB_DATABASE=admin + # - MONGO_INITDB_ROOT_USERNAME=admin + # - MONGO_INITDB_ROOT_PASSWORD=changeme + volumes: + - llmware-mongodb:/data/db:Z + ports: + - '27017:27017' + + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - llmware-etcd:/etcd:Z + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - llmware-minio:/minio_data:Z + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + milvus: + container_name: milvus + image: milvusdb/milvus:v2.3.0 + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - llmware-milvus:/var/lib/milvus:Z + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + depends_on: + - "etcd" + - "minio" + + + dev-neo4j: + container_name: devneo4j + hostname: neo4j + image: neo4j:5.15.0-community + ports: + - 7474:7474 + - 7687:7687 + restart: always + + volumes: + - $HOME/neo4j/data:/data:Z + - $HOME/neo4j/logs:/logs:Z + + environment: + - NEO4J_AUTH=none + +volumes: + llmware-mongodb: + driver: local + llmware-etcd: + driver: local + llmware-minio: + driver: local + llmware-milvus: + driver: local + + diff --git a/scripts/docker/docker-compose_mongo_milvus.yaml b/scripts/docker/docker-compose_mongo_milvus.yaml new file mode 100644 index 0000000..c7356db --- /dev/null +++ b/scripts/docker/docker-compose_mongo_milvus.yaml @@ -0,0 +1,83 @@ +version: "3.5" + +services: + mongodb: + container_name: mongodb + image: mongo:5.0.10 + # To secure MongoDB, uncomment and set the following values + # environment: + # - MONGO_INITDB_DATABASE=admin + # - MONGO_INITDB_ROOT_USERNAME=admin + # - MONGO_INITDB_ROOT_PASSWORD=changeme + volumes: + - llmware-mongodb:/data/db + ports: + - '27017:27017' + + etcd: + container_name: milvus-etcd + image: quay.io/coreos/etcd:v3.5.5 + environment: + - ETCD_AUTO_COMPACTION_MODE=revision + - ETCD_AUTO_COMPACTION_RETENTION=1000 + - ETCD_QUOTA_BACKEND_BYTES=4294967296 + - ETCD_SNAPSHOT_COUNT=50000 + volumes: + - llmware-etcd:/etcd + command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd + healthcheck: + test: ["CMD", "etcdctl", "endpoint", "health"] + interval: 30s + timeout: 20s + retries: 3 + + minio: + container_name: milvus-minio + image: minio/minio:RELEASE.2023-03-20T20-16-18Z + environment: + MINIO_ACCESS_KEY: minioadmin + MINIO_SECRET_KEY: minioadmin + ports: + - "9001:9001" + - "9000:9000" + volumes: + - llmware-minio:/minio_data + command: minio server /minio_data --console-address ":9001" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + milvus: + container_name: milvus + image: milvusdb/milvus:v2.3.0 + command: ["milvus", "run", "standalone"] + environment: + ETCD_ENDPOINTS: etcd:2379 + MINIO_ADDRESS: minio:9000 + volumes: + - llmware-milvus:/var/lib/milvus + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"] + interval: 30s + start_period: 90s + timeout: 20s + retries: 3 + ports: + - "19530:19530" + - "9091:9091" + depends_on: + - "etcd" + - "minio" + +volumes: + llmware-mongodb: + driver: local + llmware-etcd: + driver: local + llmware-minio: + driver: local + llmware-milvus: + driver: local + diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e2adc52 --- /dev/null +++ b/setup.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import os +import platform +import re +import sys +from setuptools import find_packages, setup, Extension +from pathlib import Path + +VERSION_FILE = "llmware/__init__.py" +with open(VERSION_FILE, encoding='utf-8') as version_file: + match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",version_file.read(), re.MULTILINE) + +if match: + version = match.group(1) +else: + raise RuntimeError(f"Unable to find version string in {VERSION_FILE}.") + +with open("README.md", encoding='utf-8') as readme_file: + long_description = readme_file.read() + + +def glob_fix(package_name, glob): + # this assumes setup.py lives in the folder that contains the package + package_path = Path(f'./{package_name}').resolve() + return [str(path.relative_to(package_path)) + for path in package_path.glob(glob)] + +setup( + name="llmware", # Required + version=version, # Required + description="An enterprise-grade LLM-based development framework, tools, and fine-tuned models", # Optional + long_description=long_description, # Optional + long_description_content_type="text/markdown", # Optional + url="https://github.com/llmware-ai", + project_urls={ + 'Repository': 'https://github.com/llmware-ai/llmware', + }, + author="llmware", + author_email="support@aibloks.com", # Optional + classifiers=[ # Optional + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Topic :: Software Development", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13" + ], + keywords="ai,llm,rag,data,development", # Optional + packages=['llmware'], + package_data={'llmware': ['*.c', '*.so', '*.dylib', '.dylibs/*', *glob_fix('llmware', 'lib/**/*')], 'llmware.libs': ['*']}, + python_requires=">=3.9", + zip_safe=True, + install_requires=[ + 'huggingface-hub>=0.19.4', + 'numpy>=1.23.2', + 'requests>=2.32.0', + 'tokenizers>=0.15.0', + 'boto3>=1.24.53', + 'colorama==0.4.6' + ], + + extras_require={ + 'milvus': ['pymilvus<=2.5.1'], + 'chromadb': ['chromadb>=0.4.22'], + 'pinecone': ['pinecone-client==3.0.0'], + 'lancedb': ['lancedb==0.5.0'], + 'qdrant': ['qdrant-client==1.7.0'], + 'whisper': ['soundfile>=0.12.0', 'soxr>=0.5.0'], + 'postgres': ['psycopg-binary==3.1.17', 'psycopg==3.1.17', 'pgvector==0.2.4'], + 'redis': ['redis==5.0.1'], + 'mongo': ['pymongo>=4.7.0'], + 'neo4j': ['neo4j==5.16.0'], + 'full': ['pymongo>=4.7.0', 'torch>=1.13.1', 'transformers>=4.36.0', 'einops>=0.7.0', + 'Wikipedia-API>=0.6.0','openai>=1.0', 'datasets>=2.15.0', 'yfinance>=0.2.38', 'pymilvus<=2.5.1', + 'chromadb>=0.4.22', 'streamlit', 'psycopg-binary==3.1.17', 'psycopg==3.1.17', + 'pgvector==0.2.4', 'soundfile>=0.12.0', 'soxr>=0.5.0'] + }, +) diff --git a/solutions/README.md b/solutions/README.md new file mode 100644 index 0000000..3467042 --- /dev/null +++ b/solutions/README.md @@ -0,0 +1,19 @@ +# 🔥 Solutions with LLMWare 🔥 + +New to LLMWare - [**Fast Start tutorial series**](https://github.com/llmware-ai/llmware/tree/main/tutorials) + +| Solution | Detail | +| :--------- | :--------------------------------------------------------------------------------------------------------------------- | +| 1.**RAG** | A guided series of examples on building RAG Solutions | +| 2. **Embeddings** | Building embeddings on vector DBs with wide range of models and document sources | +| 3. **Sources** | Parsing documents, extracting information, retrieval and prompt building | +| 4. **Models** | Exploring different models and pipelines | +| 5. **OpenVINO** | Guide and examples for integrating OpenVINO +| 6. **ONNXRuntime** | Guide and examples for integrating ONNXRuntime models | +| 7. **GGUF** | Guide and examples for integrating GGUF models | +| 8. **SLIM Agents** | Using small language models as specialized function-calling agents | +| 9. **UI** | Adding various UI elements | +| 10. **Use Cases** | Various end-to-end demo-ready use case scenarios | + + +Check back from time-to-time as we are always updating these examples - especially with new use cases and contributions from the llmware Community! diff --git a/solutions/embeddings/docs2vecs_with_milvus-contracts.py b/solutions/embeddings/docs2vecs_with_milvus-contracts.py new file mode 100644 index 0000000..dc99ff6 --- /dev/null +++ b/solutions/embeddings/docs2vecs_with_milvus-contracts.py @@ -0,0 +1,84 @@ + +""" This example illustrates parsing, text chunking, embedding and then querying ~80 legal documents. The +example was originally developed for a joint webinar hosted with Milvus. Please feel free to substitute +other vector databases in the example, if you prefer, along with changing the text collection DB from Mongo to +either SQLite or Postgres. + + The example uses sample documents (~80 legal template contracts) that can be pulled down with the command: + sample_files_path = Setup().load_sample_files() +""" + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.status import Status +from llmware.configs import LLMWareConfig + + +def parse_and_generate_vector_embeddings(library_name): + + # Step 0 - Configuration - we will use these in Step 4 to install the embeddings + embedding_model = "industry-bert-contracts" + vector_db = "milvus" + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses all of the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "AgreementsLarge")) + + # Step 4 - Install the embeddings + print("\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}".format(vector_db, embedding_model)) + + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + # Step 5 - start using the new vector embeddings with Query + sample_query = "incentive compensation" + print("\n\nupdate: Step 5 - Query: {}".format(sample_query)) + + query_results = Query(library).semantic_query(sample_query, result_count=20) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # for display purposes only, we will only show the first 100 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("mongo") + + # pick any name for the library + user_selected_name = "contracts" + parse_and_generate_vector_embeddings(user_selected_name) + diff --git a/solutions/embeddings/docs2vecs_with_milvus-rag.py b/solutions/embeddings/docs2vecs_with_milvus-rag.py new file mode 100644 index 0000000..2b4381d --- /dev/null +++ b/solutions/embeddings/docs2vecs_with_milvus-rag.py @@ -0,0 +1,99 @@ + +""" This example illustrates parsing, text chunking, embedding and then executing in a RAG prompt process using +~80 legal documents. The example was originally developed for a joint webinar hosted with Milvus. + + Please feel free to substitute other vector databases in the example, if you prefer. + + The example uses sample documents (~80 legal template contracts) that can be pulled down with the command: + sample_files_path = Setup().load_sample_files() +""" + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.status import Status +from llmware.prompts import Prompt +from llmware.configs import LLMWareConfig + + +def rag (library_name): + + # Step 0 - Configuration - we will use these in Step 4 to install the embeddings + embedding_model = "industry-bert-contracts" + vector_db = "milvus" + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + contracts_path = os.path.join(sample_files_path, "Agreements") + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses all of the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + library.add_files(input_folder_path=contracts_path) + + # Step 4 - Install the embeddings + print("\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}".format(vector_db, embedding_model)) + + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + print("\nupdate: Loading 1B parameter BLING model for LLM inference") + + prompter = Prompt().load_model("llmware/bling-1b-0.1") + query = "what is the executive's base annual salary" + + results = Query(library).semantic_query(query, result_count=50, embedding_distance_threshold=1.0) + + for i, res in enumerate(results): + + print("update: ", i, res["file_source"], res["distance"], res["text"]) + + for i, contract in enumerate(os.listdir(contracts_path)): + + qr = [] + + if contract != ".DS_Store": + + print("\nContract Name: ", i, contract) + + for j, entries in enumerate(results): + if entries["file_source"] == contract: + print("Top Retrieval: ", j, entries["distance"], entries["text"]) + qr.append(entries) + + source = prompter.add_source_query_results(query_results=qr) + response = prompter.prompt_with_source(query, prompt_name="default_with_context", temperature=0.3) + + for resp in response: + if "llm_response" in resp: + print("\nupdate: llm answer - ", resp["llm_response"]) + + # start fresh for next document + prompter.clear_source_materials() + + +if __name__ == "__main__": + + # feel free to change to sqlite or postgres (if installed) + LLMWareConfig().set_active_db("mongo") + + # pick any name for the library + user_selected_name = "contracts_rag10" + rag(user_selected_name) + + diff --git a/solutions/embeddings/docs2vecs_with_milvus-un_resolutions.py b/solutions/embeddings/docs2vecs_with_milvus-un_resolutions.py new file mode 100644 index 0000000..af7da7f --- /dev/null +++ b/solutions/embeddings/docs2vecs_with_milvus-un_resolutions.py @@ -0,0 +1,84 @@ + +""" This example demonstrates parsing, text chunking and embedding with 500 United Nations (UN) Resolutions. +The sample documents (500 PDFs - 2-15 pages each) can be pulled down from a public S3 repo with the command: + sample_files_path = Setup().load_sample_files() + + note: the example assumes that you have installed Milvus and MongoDB per the separate instructions in the README + +""" + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.status import Status +from llmware.configs import LLMWareConfig + + +def parse_and_generate_vector_embeddings(library_name): + + # Step 0 - Configuration - we will use these in Step 4 to install the embeddings + embedding_model = "mini-lm-sbert" + vector_db = "milvus" + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses all of the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "UN-Resolutions-500")) + + # Step 4 - Install the embeddings + print("\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}".format(vector_db, embedding_model)) + + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + # Step 5 - start using the new vector embeddings with Query + sample_query = "sustainability issues impacting women" + print("\n\nupdate: Step 5 - Query: {}".format(sample_query)) + + query_results = Query(library).semantic_query(sample_query, result_count=20) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # for display purposes only, we will only show the first 100 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("mongo") + + # pick any name for the library + user_selected_name = "un_resolutions500" + parse_and_generate_vector_embeddings(user_selected_name) + + diff --git a/solutions/embeddings/embeddings_fast_start.py b/solutions/embeddings/embeddings_fast_start.py new file mode 100644 index 0000000..7247b60 --- /dev/null +++ b/solutions/embeddings/embeddings_fast_start.py @@ -0,0 +1,72 @@ + +""" This example shows the general recipe for creating an embedding. This scenario uses ChromaDB in local + file mode for no-install laptop deployment. + + NOTE: you may need to install separately: pip3 install chromadb. + +""" + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from importlib import util +from llmware.configs import LLMWareConfig + +if not util.find_spec("chromadb"): + print("\nto run this example with chromadb, you need to install the chromadb python sdk: pip3 install chromadb") + + +def embeddings_fast_start (library_name, vector_db="chromadb"): + + # Create and populate a library + print (f"\nstep 1 - creating and populating library: {library_name}...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "AgreementsLarge")) + + # To create vector embeddings you just need to specify the embedding model and the vector embedding DB + # For examples of using HuggingFace and SentenceTransformer models, see those examples in this same folder + + embedding_model = "mini-lm-sbert" + + print (f"\n > Generating embedding vectors and storing in '{vector_db}'...") + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db) + + # Then when doing semantic queries, the most recent vector DB used for embeddings will be used. + + # We just find the best 3 hits for "Salary" + q = Query(library) + print (f"\n > Running a query for 'Salary'...") + query_results = q.semantic_query(query="Salary", result_count=10, results_only=True) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # for display purposes only, we will only show the first 100 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + return query_results + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + # set to 'chromadb' local file storage for no-install fast start + db = "chromadb" + embeddings_fast_start("embedding_test_1", vector_db=db) + + diff --git a/solutions/embeddings/using_chromadb.py b/solutions/embeddings/using_chromadb.py new file mode 100644 index 0000000..c697e21 --- /dev/null +++ b/solutions/embeddings/using_chromadb.py @@ -0,0 +1,121 @@ + +"""This example shows how to use ChromaDB as a vector embedding database with llmware with the + default configuration of using ChromaDB as a local persistent file-based vector db, with options + for both in-memory and client-server installations. + + (A) Python Dependencies - + + As a first step, confirm that you have installed chromadb, e.g., `pip3 install chromadb` + + (B) Using ChromaDB - + + Installing ChromaDB via pip installs everything you need. + However, if you need help, there are many great online sources and communities, e.g.,: + -- ChromaDB documentation - https://docs.trychroma.com/ + -- Docker - https://hub.docker.com/u/chromadb + -- please also see the docker-compose-chromadb.yaml script provided in the llmware script repository + + (C) Configurations - + + You can configure ChromaDB with environment variables. Here is the list of variable names we currently + support - for more information see ChromaDBConfig. + -- CHROMADB_HOST + -- CHROMADB_PORT + -- CHROMADB_SSL + -- CHROMADB_HEADERS + -- CHROMADB_SERVER_AUTH_PROVIDER + -- CHROMADB_SERVER_AUTH_CREDENTIALS_PROVIDER + -- CHROMADB_SERVER_AUTH_CREDENTIALS_PROVIDER + -- CHROMADB_PASSWORD + -- CHROMADB_SERVER_AUTH_CREDENTIALS_FILE + -- CHROMADB_SERVER_AUTH_CREDENTIALS + -- CHROMADB_SERVER_AUTH_TOKEN_TRANSPORT_HEADER +""" + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig, ChromaDBConfig + + +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder), + chunk_size=400, max_chunk_size=600, smart_chunking=1) + + return library + + +# start script + +if __name__ == "__main__": + + # configs + LLMWareConfig().set_active_db("sqlite") + library_name = "chromadb_lib_1" + + print("update: chromadb - persistent path - ", ChromaDBConfig().get_config("persistent_path")) + + print("update: Step 1- starting here- building library- parsing PDFs into text chunks") + + lib = build_lib(library_name) + + # after building the library the first time, you can skip that step, and load the library directly by + # uncommenting the line below + # lib = Library().load_library(library_name) + + # optional - check the status of the library card and embedding + lib_card = lib.get_library_card() + print("update: -- before embedding process - check library card - ", lib_card) + + print("update: Step 2 - starting to install embeddings") + + # alt embedding models - "mini-lm-sbert" | industry-bert-contracts | text-embedding-ada-002 + # note: if you want to use text-embedding-ada-002, you will need an OpenAI key and enter into os.environ variable + # e.g., os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # batch sizes from 100-500 usually give good performance and work on most environments + lib.install_new_embedding(embedding_model_name="industry-bert-contracts",vector_db="chromadb",batch_size=100) + + # optional - check the status of the library card and embedding + lib_card = lib.get_library_card() + print("update: -- after embedding process - check updated library card - ", lib_card) + + # run a query + # note: embedding_model_name is optional, but useful if you create multiple embeddings on the same library + # --see other example scripts for multiple embeddings + + # create query object + query_chromadb = Query(lib) + + # run multiple queries using query_chromadb + my_search_results = query_chromadb.semantic_query("What is the sale bonus?", result_count = 24) + + for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + + # if you want to delete the embedding - uncomment the line below + # lib.delete_installed_embedding("industry-bert-contracts", "chromadb") + + # optional - check the embeddings on the library + emb_record = lib.get_embedding_status() + for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) diff --git a/solutions/embeddings/using_lance.py b/solutions/embeddings/using_lance.py new file mode 100644 index 0000000..ff4bdee --- /dev/null +++ b/solutions/embeddings/using_lance.py @@ -0,0 +1,74 @@ + +"""This example demonstrates creating vector embeddings (used for doing semantic queries) + Note: lancedb is not used in the example below as it requires an API key. If you have a lancedb account, you can set these two variables: + os.environ.get("USER_MANAGED_lancedb_API_KEY") = + os.environ.get("USER_MANAGED_lancedb_ENVIRONMENT") = (for example "gcp-starter") +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def embeddings_lancedb (library_name): + + # Create and populate a library + print (f"\nstep 1 - creating and populating library: {library_name}...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements")) + + # To create vector embeddings you just need to specify the embedding model and the vector embedding DB + # For examples of using HuggingFace and SentenceTransformer models, see those examples in this same folder + + embedding_model = "mini-lm-sbert" + + print (f"\n > Generating embedding vectors and storing in lancedb ...") + + # note: the only code change to use a different vector_db is changing the name in this method below + library.install_new_embedding(embedding_model_name=embedding_model, vector_db="lancedb") + + # Then when doing semantic queries, the most recent vector DB used for embeddings will be used. + + # We just find the best 3 hits for "Salary" + q = Query(library) + print (f"\n > Running a query for 'Salary'...") + query_results = q.semantic_query(query="Salary", result_count=10, results_only=True) + + print("query-",query_results) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # for display purposes only, we will only show the first 100 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + return query_results + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + library_name = "embedding_test_0" + + # note: these two environmental variables will be checked to apply your lancedb keys + # os.environ["USER_MANAGED_lancedb_API_KEY"] = "your-lancedb-api-key" + # os.environ["USER_MANAGED_lancedb_ENVIRONMENT"] = "your-lancedb-environment" + + embeddings_lancedb("embedding_test") + + diff --git a/solutions/embeddings/using_milvus_lite.py b/solutions/embeddings/using_milvus_lite.py new file mode 100644 index 0000000..4a4e718 --- /dev/null +++ b/solutions/embeddings/using_milvus_lite.py @@ -0,0 +1,145 @@ + +""" This example is a fast start with Milvus Lite, which is a 'no-install' file-based version of Milvus, intended +for rapid prototyping. A couple of key points to note: + + -- Platform - per Milvus docs, Milvus Lite is designed for Mac and Linux (not on Windows currently) + -- PyMilvus - need to `pip install pymilvus>=2.4.2` + -- within LLMWare: set MilvusConfig().set_config("lite", True) +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.status import Status +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig, MilvusConfig + +from importlib import util + +if not util.find_spec("pymilvus"): + print("\nto run this example with pymilvus, you need to install pymilvus: pip3 install pymilvus>=2.4.2") + + +def setup_library(library_name): + + """ Note: this setup_library method is provided to enable a self-contained example to create a test library """ + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # check the embedding status 'before' installing the embedding + embedding_record = library.get_embedding_status() + print("embedding record - before embedding ", embedding_record) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in database + + print("update: Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements"), + chunk_size=400, max_chunk_size=600, smart_chunking=1) + + return library + + +def install_vector_embeddings(library, embedding_model_name): + + """ This method is the core example of installing an embedding on a library. + -- two inputs - (1) a pre-created library object and (2) the name of an embedding model """ + + library_name = library.library_name + vector_db = LLMWareConfig().get_vector_db() + + print(f"\nupdate: Starting the Embedding: " + f"library - {library_name} - " + f"vector_db - {vector_db} - " + f"model - {embedding_model_name}") + + # *** this is the one key line of code to create the embedding *** + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db,batch_size=100) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + # Start using the new vector embeddings with Query + sample_query = "incentive compensation" + print("\n\nupdate: Run a sample semantic/vector query: {}".format(sample_query)) + + # queries are constructed by creating a Query object, and passing a library as input + query_results = Query(library).semantic_query(sample_query, result_count=20) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # to see all of the dictionary keys returned, uncomment the line below + # print("update: query_results - all - ", i, entries) + + # for display purposes only, we will only show the first 125 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + # lets take a look at the library embedding status again at the end to confirm embeddings were created + embedding_record = library.get_embedding_status() + + print("\nupdate: embedding record - ", embedding_record) + + return 0 + + +if __name__ == "__main__": + + # Fast Start configuration - will use no-install embedded sqlite + # -- if you have installed Mongo or Postgres, then change the .set_active_db accordingly + + LLMWareConfig().set_active_db("sqlite") + + # set the "lite" flag in MilvusConfig to True -> to use server version, set to False (which is default) + MilvusConfig().set_config("lite", True) + LLMWareConfig().set_vector_db("milvus") + + # Step 1 - create library + library = setup_library("ex2_milvus_lite") + + # Step 2 - Select any embedding model in the LLMWare catalog + + # to see a list of the embedding models supported, uncomment the line below and print the list + embedding_models = ModelCatalog().list_embedding_models() + + # for i, models in enumerate(embedding_models): + # print("embedding models: ", i, models) + + # for this first embedding, we will use a very popular and fast sentence transformer + embedding_model = "mini-lm-sbert" + + # note: if you want to swap out "mini-lm-sbert" for Open AI 'text-embedding-ada-002', uncomment these lines: + # embedding_model = "text-embedding-ada-002" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # run the core script + install_vector_embeddings(library, embedding_model) + + + + + diff --git a/solutions/embeddings/using_mongo_atlas.py b/solutions/embeddings/using_mongo_atlas.py new file mode 100644 index 0000000..5eb264e --- /dev/null +++ b/solutions/embeddings/using_mongo_atlas.py @@ -0,0 +1,71 @@ + +"""This example demonstrates working with MongoDB Atlas. You can use Mongo Atlas in 2 ways: + +1. Only as a vector DB (and use local or another mongo for data storage) + In this case, you can set the following environment variable: + MONGO_ATLAS_CONNECTION_URI=mongo+srv://:@... + +2. For both vector and data storage + In this case, you can set the following environment configuration: + LLMWareConfig().set_config("collection_db_uri", "mongo+srv://:@..." + +This example demonstrates the 2nd approach. + +""" + +import os +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +# Use MongoDB Atlas for both data storage and vector embedding storage/search + + +def using_mongo_atlas(mongo_atlas_connection_string): + + LLMWareConfig().set_config("collection_db_uri", mongo_atlas_connection_string) + + # Create a library and populate it with some sample documents + + library_name = "test_mongo_atlas" + print(f"\n > Creating library {library_name}...") + + library = Library().create_new_library(library_name) + print(f"\n > Loading the llmware Sample Files...") + + sample_files_path = Setup().load_sample_files() + print(f"\n > Adding some files to the library...") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements")) + + # Create vector embeddings using Mongo Atlas + print(f"\n > Generating embedding vectors (using the 'mini-lm-sbert' model) and storing them (in 'Mongo Atlas')...") + library.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="mongo_atlas") + + # Do a semantic search in the library + print(f"\n > Running a query for 'Salary'...") + query_results = Query(library).semantic_query(query="salary", result_count=10, results_only=True) + + print(f"\n\nResults for 'Salary' in {library_name}:\n") + for query_result in query_results: + print( + "File: " + query_result["file_source"] + " (Page " + str(query_result["page_num"]) + "):\n" + query_result[ + "text"] + "\n") + + return query_results + + +if __name__ == "__main__": + + # Set this env var appropriately or just paste in your Mongo Atlas connection string: + mongo_config = os.environ["MONGO_ATLAS_CONNECTION_URI"] + + output = using_mongo_atlas(mongo_config) + + + + + + diff --git a/solutions/embeddings/using_multiple_embeddings.py b/solutions/embeddings/using_multiple_embeddings.py new file mode 100644 index 0000000..02ba8f0 --- /dev/null +++ b/solutions/embeddings/using_multiple_embeddings.py @@ -0,0 +1,158 @@ + +"""This example shows how to easily create multiple embeddings over the same library with llmware + + This recipe can be especially useful when trying to compare the effectiveness of a particular + embedding model for a specific domain or library corpus and to run other comparative experiments + without being 'locked-in' to a particular model. + + Note: the example uses four different embedding models: + + 1. mini-lm-sbert - a favorite small, fast Sentence Transformer included in the llmware model catalog by default + 2. text-embedding-ada-002 - the popular OpenAI embedding model + 3. industry-bert-sec - an industry fine-tuned embedding model, in the llmware model catalog + 4. all-mpnet-base-v2 - one of the most popular Sentence Transformers (which we will register and add to the + model catalog on the fly + + To use OpenAI Ada will require an Open API key - if you do not have one, feel free to comment out or + select a different model. Any Sentence Transformer or Huggingface embedding model can be used. + +""" + + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + + +os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + +os.environ["TOKENIZERS_PARALLELISM"] = "false" # Avoid a HuggingFace tokenizer warning + + +# Note: this will build a small library that will be used in the embedding examples +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: downloading sample files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: parsing and text indexing files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +# use multiple embedding models on the same library and the same vector db + +def multiple_embeddings_same_db_same_lib(document_folder=None,sample_query=None,vector_db=None, base_library_name=None): + + print("\nupdate: Step 1- starting here- building library- parsing PDFs into text chunks") + + lib = build_lib(base_library_name, folder=document_folder) + + # optional - check the status of the library card and embedding + lib_card = lib.get_library_card() + print("update: library card - ", lib_card) + + print("\nupdate: Step 2 - starting to install embeddings") + + # alt embedding models - "mini-lm-sbert" | industry-bert-contracts | text-embedding-ada-002 + # note: if you want to use text-embedding-ada-002, you will need an OpenAI key and enter into os.environ variable + # e.g., os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # Note: batch size can be configured based on memory of machine and optimized for performance + # -- generally, between 100-500 is a safe range to optimize performance/memory + + print(f"\nupdate: Embedding #1 - mini-lm-sbert - {vector_db}") + lib.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db=vector_db, batch_size=200) + + print(f"\nupdate: Embedding #2 - text-embedding-ada-002 - {vector_db}") + lib.install_new_embedding(embedding_model_name="text-embedding-ada-002", vector_db=vector_db, batch_size=500) + + print(f"\nupdate: Embedding #3 - industry-bert-sec - {vector_db}") + lib.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db=vector_db, batch_size=100) + + # for the last embedding, we will register a popular open source sentence transformer model to use + # -- see "using_sentence_transformer.py" for more details + + ModelCatalog().register_sentence_transformer_model(model_name="all-mpnet-base-v2", + embedding_dims=768, context_window=384) + + # use directly now as an embedding model + print(f"\nupdate: Embedding #4 - all-mpnet-base-v2 - {vector_db}") + lib.install_new_embedding(embedding_model_name="all-mpnet-base-v2",vector_db=vector_db,batch_size=300) + + # optional - check the embeddings on the library + print("\nupdate: Embedding record of the Library") + + emb_record = lib.get_embedding_status() + for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + + # Using the Embeddings to Execute Queries + # + # create query object: + # 1. if no embedding_model or vector_db passed in constructor, then selects the LAST embedding record, which + # is the most recent embedding on the library, and uses that combination of model + vector db + # + # 2. if embedding_model_name only passed, then looks up the first instance of that embedding model + # in the embedding record, and will use the associated vector db + # + # 3. if both embedding_model_name and vector_db passed in constructor, then looks up that combo in + # embedding record. + + query1 = Query(lib, embedding_model_name="mini-lm-sbert") + query2 = Query(lib, embedding_model_name="text-embedding-ada-002") + + # to execute query against any of the query objects + minilm_results = query1.semantic_query(sample_query, result_count=12) + ada_results = query2.semantic_query(sample_query, result_count=12) + + print("\n\nupdate: Sample Query using Embeddings") + + print("\nupdate: Embedding Model # 1 - MiniLM SBERT Results") + for i, qr1 in enumerate(minilm_results): + print("update: minilm semantic query results: ", i, qr1["distance"], qr1) + + print("\nupdate: Embedding Model # 2- Ada Results") + for j, qr2 in enumerate(ada_results): + print("update: ada semantic query results: ", j, qr2["distance"], qr2) + + return 0 + + +if __name__ == "__main__": + + # document folder options: Agreements | UN-Resolutions-500 + # note: Agreements = ~15 contracts = ~1272 embeddings - takes ~5 minutes to run (without GPU) + # note: UN-Resolutions-500 = 500 documents = ~12500 embeddings - takes ~15-20 minutes to run (without GPU) + # -- good sample query for UN-Resolutions, e.g. "what are key initiatives to promote sustainability?" + # + # try substituting different vector-db, e.g, "pg_vector" | "redis" | "faiss" + + # please note: to use multiple embeddings on a library requires either Mongo or Postgres as text collection + + LLMWareConfig().set_active_db("mongo") + + multiple_embeddings_same_db_same_lib(document_folder="Agreements", + sample_query="what is the sale bonus?", + vector_db="milvus", + base_library_name="multi_embeddings_test_lib_0") + + + diff --git a/solutions/embeddings/using_multiple_vector_db.py b/solutions/embeddings/using_multiple_vector_db.py new file mode 100644 index 0000000..7b64475 --- /dev/null +++ b/solutions/embeddings/using_multiple_vector_db.py @@ -0,0 +1,139 @@ + +"""This example shows how to use a single library and attach 'mix-and-match' multiple vector databases with + potentially multiple different embedding models + + Use case: evaluate and compare multiple combinations of vector databases and embedding models using the + same core text library - without having to recreate - enables fast experimentation without lock-in. + + Note: + -- the example belows requires installation of several vector db- Milvus, PGVector, Redis and FAISS + -- docker-compose scripts for rapid install for Milvus, PGVector and Redis in the llmware repository + -- no install required for FAISS + -- please also see the install instructions in the Examples/Embeddings for more install pre-reqs, e.g.,: + --Milvus: pip install pymilvus + --Redis-Stack-Server: pip install redis + --Postgres: pip install psycopg-binary psycopg pgvector + """ + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + +os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + +os.environ["TOKENIZERS_PARALLELISM"] = "false" # Avoid a HuggingFace tokenizer warning + + +# Note: this will build a small library that will be used in the embedding examples +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: downloading sample files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: parsing and text indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +def multiple_embeddings_and_multiple_vector_dbs(document_folder=None,sample_query="", base_library_name=""): + + print("\nupdate: Step 1- starting here- building library- parsing PDFs into text chunks") + + lib = build_lib(base_library_name, folder=document_folder) + + # optional - check the status of the library card and embedding + lib_card = lib.get_library_card() + print("update: library card - ", lib_card) + + print("\nupdate: Step 2 - starting to install embeddings") + + # mix-and-match with different embedding models and vector db on same library content + + # We will create 6 different embeddings across 4 different vector databases - using the same library + # note: you can run many different models on the same db, or the same model across multiple dbs + + print("\nupdate: Embedding #1 - industry-bert-contracts - on PG_Vector") + lib.install_new_embedding(embedding_model_name="industry-bert-contracts",vector_db="pg_vector",batch_size=300) + + print("\nupdate: Embedding #2 - mini-lm-sbert - Milvus") + lib.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="milvus", batch_size=200) + + print("\nupdate: Embedding #3 - mini-lm-sbert - on PG_Vector") + lib.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="pg_vector", batch_size=100) + + print("\nupdate: Embedding #4 - text-embedding-ada-002 - FAISS") + lib.install_new_embedding(embedding_model_name="text-embedding-ada-002", vector_db="faiss", batch_size=500) + + print("\nupdate: Embedding #5 - industry-bert-sec - REDIS") + lib.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="redis", batch_size=350) + + # for the last embedding, we will register a pretrained sentence transformer model to use + # -- see "using_sentence_transformer.py" for more details + ModelCatalog().register_sentence_transformer_model(model_name= "all-MiniLM-L6-v2", + embedding_dims=384, context_window=256) + + # use directly now as an embedding model + print("\nupdate: Embedding #6 - all-MiniLM-L6-v2 - REDIS") + lib.install_new_embedding(embedding_model_name="all-MiniLM-L6-v2",vector_db="redis",batch_size=300) + + # optional - check the embeddings on the library + print("\nupdate: Embedding record of the Library") + emb_record = lib.get_embedding_status() + for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + + # Using the Embeddings to Execute Queries + # + # create query object: + # 1. if no embedding_model or vector_db passed in constructor, then selects the LAST embedding record, which + # is the most recent embedding on the library, and uses that combination of model + vector db + # + # 2. if embedding_model_name only passed, then looks up the first instance of that embedding model + # in the embedding record, and will use the associated vector db + # + # 3. if both embedding_model_name and vector_db passed in constructor, then looks up that combo in + # embedding record + + q = Query(lib, embedding_model_name="mini-lm-sbert", vector_db="pg_vector") + + # to execute query against any of the query objects: + # --just showing one example + my_search_results = q.semantic_query(sample_query, result_count=15) + + print("\n\nupdate: Sample Query using Embeddings") + + for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + + # if you want to delete any of the embeddings - uncomment the line below + # lib.delete_installed_embedding("industry-bert-contracts", "pg_vector") + + return 0 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("mongo") + + multiple_embeddings_and_multiple_vector_dbs(document_folder="Agreements", + sample_query="what is the base salary?", + base_library_name="multi-embedding-multi-db-test-1") + + diff --git a/solutions/embeddings/using_neo4j.py b/solutions/embeddings/using_neo4j.py new file mode 100644 index 0000000..a8cca91 --- /dev/null +++ b/solutions/embeddings/using_neo4j.py @@ -0,0 +1,113 @@ + +"""This example shows how to use Neo4j as a vector embedding database with llmware + + (A) Python Dependencies - + + As a first step, you should pip install theh Neo4j driver, which is not included in the llmware package: + 1. pip3 install neo4j + + (B) Installing Neo4j - + + If you need help installing Neo4j, there are many great online sources and communities, e.g.,: + -- Neo4j Installation - https://neo4j.com/docs/operations-manual/current/installation/ (All OS) + -- Mac OS - https://neo4j.com/docs/operations-manual/current/installation/osx/ + -- Linux - https://neo4j.com/docs/operations-manual/current/installation/linux/ + -- Debian repository - https://debian.neo4j.com/ + -- Windows - https://neo4j.com/docs/operations-manual/current/installation/windows/ + -- Docker - https://hub.docker.com/_/neo4j + -- please also see the docker-compose-neo4j.yaml script provided in the llmware script repository + + (C) Configurations - + + -- set os.environ variables to 'automatically' pass in installing embedding + -- os.environ["NEO4J_URI"] = "neo4j://localhost:7687" + -- os.environ["NEO4J_USERNAME"] = "neo4j" # by default + -- os.environ["NEO4J_PASSWORD"] = "neo4j" # by default + -- os.environ["NEO4J_DATABASE"] = "llmware" + +""" + + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + +# example default Neo4j install with database = llmware & user = neo4j +os.environ["NEO4J_URI"] = "neo4j://localhost:7687" +os.environ["NEO4J_USERNAME"] = "neo4j" +os.environ["NEO4J_PASSWORD"] = "neo4j" +os.environ["NEO4J_DATABASE"] = "llmware" + + +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +# start script + +print("update: Step 1- starting here- building library- parsing PDFs into text chunks") + +LLMWareConfig().set_active_db("sqlite") + +lib = build_lib("neo4j_lib_0") + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- before embedding process - check library card - ", lib_card) + +print("update: Step 2 - starting to install embeddings") + +# alt embedding models - "mini-lm-sbert" | industry-bert-contracts | text-embedding-ada-002 +# note: if you want to use text-embedding-ada-002, you will need an OpenAI key and enter into os.environ variable +# e.g., os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + +# batch sizes from 100-500 usually give good performance and work on most environments +lib.install_new_embedding(embedding_model_name="industry-bert-contracts",vector_db="neo4j",batch_size=300) + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- after embedding process - check updated library card - ", lib_card) + +# run a query +# note: embedding_model_name is optional, but useful if you create multiple embeddings on the same library +# --see other example scripts for multiple embeddings + +# create query object +query_neo4j = Query(lib, embedding_model_name="industry-bert-contracts") + +# run multiple queries using query_neo4j +my_search_results = query_neo4j.semantic_query("What is the sale bonus?", result_count = 24) + +for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + +# if you want to delete the embedding - uncomment the line below +# lib.delete_installed_embedding("industry-bert-contracts", "neo4j") + +# optional - check the embeddings on the library +emb_record = lib.get_embedding_status() +for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + diff --git a/solutions/embeddings/using_pg_vector.py b/solutions/embeddings/using_pg_vector.py new file mode 100644 index 0000000..8b217d3 --- /dev/null +++ b/solutions/embeddings/using_pg_vector.py @@ -0,0 +1,112 @@ + +"""This example shows how to use PG Vector (Postgres) as a vector embedding database with llmware + + (A) Python Dependencies - + + As a first step, you should pip install three Postgres dependencies not included in the llmware package: + 1. pip3 install psycopg-binary + 2. pip3 install psycopg + 3. pip3 install pgvector + + (B) Installing Postgres - + + If you need help installing Postgres (with PG Vector), there are many great online sources and communities, e.g.,: + -- PostgresSQL - https://www.postgresql.org/download/ (All OS) + -- Mac OS Homebrew - https://wiki.postgresql.org/wiki/Homebrew - (brew install postgresql) + -- Linux - https://www.postgresqltutorial.com/postgresql-getting-started/install-postgresql-linux/ + -- Windows - https://www.postgresqltutorial.com/postgresql-getting-started/install-postgresql/ + -- Docker - https://www.docker.com/blog/how-to-use-the-postgres-docker-official-image/ + -- please also see the docker-compose-ankane.yaml script provided in the llmware script repository + + (C) Configurations - + + -- set os.environ variables to 'automatically' pass in installing embedding + -- os.environ["USER_MANAGED_PG_HOST"] = "localhost" + -- os.environ["USER_MANAGED_PG_PORT"] = 5432 + -- os.environ["USER_MANAGED_PG_DB_NAME"] = "postgres" # by default + -- os.environ["USER_MANAGED_PG_USER_NAME"] = "postgres" # by default + -- os.environ["USER_MANAGED_PG_PW"] = optional + +""" + + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + +# example default postgres install with database = postgres & user = postgres +os.environ["USER_MANAGED_PG_DB_NAME"] = "postgres" +os.environ["USER_MANAGED_PG_USER_NAME"] = "postgres" + +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +# start script + +LLMWareConfig().set_active_db("postgres") + +print("update: Step 1- starting here- building library- parsing PDFs into text chunks") + +lib = build_lib("pgv_lib_0") + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- before embedding process - check library card - ", lib_card) + +print("update: Step 2 - starting to install embeddings") + +# alt embedding models - "mini-lm-sbert" | industry-bert-contracts | text-embedding-ada-002 +# note: if you want to use text-embedding-ada-002, you will need an OpenAI key and enter into os.environ variable +# e.g., os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + +# batch sizes from 100-500 usually give good performance and work on most environments +lib.install_new_embedding(embedding_model_name="industry-bert-contracts",vector_db="pg_vector",batch_size=300) + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- after embedding process - check updated library card - ", lib_card) + +# run a query +# note: embedding_model_name is optional, but useful if you create multiple embeddings on the same library +# --see other example scripts for multiple embeddings + +# create query object +query_pgv = Query(lib, embedding_model_name="industry-bert-contracts") + +# run multiple queries using query_pgv +my_search_results = query_pgv.semantic_query("What is the sale bonus?", result_count = 24) + +for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + +# if you want to delete the embedding - uncomment the line below +# lib.delete_installed_embedding("industry-bert-contracts", "pg_vector") + +# optional - check the embeddings on the library +emb_record = lib.get_embedding_status() +for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + diff --git a/solutions/embeddings/using_pinecone.py b/solutions/embeddings/using_pinecone.py new file mode 100644 index 0000000..28f0e41 --- /dev/null +++ b/solutions/embeddings/using_pinecone.py @@ -0,0 +1,69 @@ + +"""This example demonstrates creating vector embeddings (used for doing semantic queries) + Note: Pinecone is not used in the example below as it requires an API key. If you have a Pinecone account, you can set these two variables: + os.environ.get("USER_MANAGED_PINECONE_API_KEY") = + os.environ.get("USER_MANAGED_PINECONE_ENVIRONMENT") = (for example "gcp-starter") +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def embeddings_pinecone (library_name): + + # Create and populate a library + print (f"\nstep 1 - creating and populating library: {library_name}...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements")) + + # To create vector embeddings you just need to specify the embedding model and the vector embedding DB + # For examples of using HuggingFace and SentenceTransformer models, see those examples in this same folder + + embedding_model = "mini-lm-sbert" + + print (f"\n > Generating embedding vectors and storing in Pinecone ...") + + # note: the only code change to use a different vector_db is changing the name in this method below + library.install_new_embedding(embedding_model_name=embedding_model, vector_db="pinecone") + + # Then when doing semantic queries, the most recent vector DB used for embeddings will be used. + + # We just find the best 3 hits for "Salary" + q = Query(library) + print (f"\n > Running a query for 'Salary'...") + query_results = q.semantic_query(query="Salary", result_count=10, results_only=True) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # for display purposes only, we will only show the first 100 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + return query_results + + +if __name__ == "__main__": + + library_name = "embedding_test_0" + + # note: these two environmental variables will be checked to apply your pinecone keys + os.environ["USER_MANAGED_PINECONE_API_KEY"] = "your-pinecone-api-key" + os.environ["USER_MANAGED_PINECONE_ENVIRONMENT"] = "your-pinecone-environment" + + embeddings_pinecone("embedding_test") + + diff --git a/solutions/embeddings/using_pinecone_openai.bash b/solutions/embeddings/using_pinecone_openai.bash new file mode 100644 index 0000000..692811f --- /dev/null +++ b/solutions/embeddings/using_pinecone_openai.bash @@ -0,0 +1,10 @@ +# This script shows how you can set the environemtn variables for the Pinecone OpenAI example. +# +# run this script with the command source, for example +# source ./using_pinecone_openai.bash + +export USER_MANAGED_PINECONE_API_KEY= +export USER_MANAGED_PINECONE_CLOUD='aws' +export USER_MANAGED_PINECONE_REGION='us-west-2' + +export USER_MANAGED_OPENAI_API_KEY= diff --git a/solutions/embeddings/using_pinecone_openai.py b/solutions/embeddings/using_pinecone_openai.py new file mode 100644 index 0000000..b879098 --- /dev/null +++ b/solutions/embeddings/using_pinecone_openai.py @@ -0,0 +1,146 @@ +'''This embedding example shows how you can use llmware in combination with OpenAI embedding models to create +a library that you can query semantically. + +This example script can be easily extended towards RAG. You can, for exmaple, create a function +that reveices the result from the query as context for a LLM to generate an answer. +''' +import os +import logging + + +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +logging.basicConfig(level = logging.INFO) +logger = logging.getLogger('llmware-pinecone-openai') + + +'''Change the values below to your API keys, and the cloud and region you want to use. If you want to use the +bash script, then you have to comment out the following code lines. + +See the Pinecone documentation for details on available cloud and region options. During testing, we used +'aws' for cloud and 'us-west-2' for region. +''' +os.environ['USER_MANAGED_PINECONE_API_KEY'] = '' +os.environ['USER_MANAGED_PINECONE_CLOUD'] = '' +os.environ['USER_MANAGED_PINECONE_REGION'] = '' + +os.environ['USER_MANAGED_OPENAI_API_KEY'] = '' + + + +def set_up_api_keys( + pinecone_api_key=os.getenv('USER_MANAGED_PINECONE_API_KEY', None), + pinecone_cloud=os.getenv('USER_MANAGED_PINECONE_CLOUD', None), + pinecone_region=os.getenv('USER_MANAGED_PINECONE_REGION', None), + openai_api_key=os.getenv('USER_MANAGED_OPENAI_API_KEY', None)): + '''This function sets the API keys for Pinecone and OpenAI, they have to be set! + ''' + logger.info('Setting up Pinecone and OpenAI API keys') + + if pinecone_api_key in [None, '']: + raise ValueError(f'You need to set the pinecone API key, got {pinecone_api_key}') + + if pinecone_cloud in [None, '']: + raise ValueError(f'You need to set the pinecone cloud, got {pinecone_environment}') + + if pinecone_region in [None, '']: + raise ValueError(f'You need to set the pinecone cloud, got {pinecone_region}') + + if openai_api_key in [None, '']: + raise ValueError(f'You need to set the OpenAI API key, got {openai_api_key}') + + + os.environ.setdefault('USER_MANAGED_PINECONE_API_KEY', pinecone_api_key) + os.environ.setdefault('USER_MANAGED_PINECONE_CLOUD', pinecone_cloud) + os.environ.setdefault('USER_MANAGED_PINECONE_REGION', pinecone_region) + + os.environ.setdefault('USER_MANAGED_OPENAI_API_KEY', openai_api_key) + + +def set_up_agreements(): + '''This function makes sure that the sample files are loaded, and returns the path the Agreements + folter. We need the path to the agreements folder for the ``Library`` object. + + If you have your own data, simply exchange this function with another one that returns a path + to you sample files. + ''' + logger.info('Setting up Aggreements') + + sample_files_path = Setup().load_sample_files() + return os.path.join(sample_files_path, "Agreements") + + +def set_up_library( + input_folder_path, + library_name='example_pinecone_openai'): + '''This function creates the library with name ``library_name`` from ``directory``. + ''' + logger.info(f'Setting up library with name {library_name} from directory {input_folder_path}') + + library = Library().create_new_library(library_name) + library.add_files(input_folder_path=input_folder_path) + return library + + +def set_up_embeddings( + library, + embedding_model='text-embedding-ada-002'): + '''This function sets up the embeddings in ``library`` with the model ``embedding_model``. + + If you bring your own data and this data contains text and images, than you need to change ``embedding_model`` + to one that can process both simultanously. + ''' + logger.info(f'Setting up embeddings in library {library.library_name} with model {embedding_model}') + + library.install_new_embedding(embedding_model_name=embedding_model, vector_db="pinecone") + return library + + +def query_library( + library, + semantic_query='Salary'): + '''This function executes the semantic query ``query`` on ``library``. + + If you want to query for something else, simply overwrite ``semantic_query``. + ''' + query = Query(library) + query_results = query.semantic_query(query=semantic_query, result_count=10, results_only=True) + + for idx, query_result in enumerate(query_results): + # each query result is a dictionary with many useful keys + text = query_result['text'] + file_source = query_result['file_source'] + page_num = query_result['page_num'] + distance = query_result['distance'] + + # We truncate the text because we want to only show a peak. + if len(text) > 125: text = f'{text[0:125]} ...' + + logger.info(f'{idx} query result: {distance} distance '\ + f'from file {file_source} and page number {page_num}, '\ + f'here is sample text:\n{text}') + + return query_results + + +def main(): + '''This function first sets the environment variables for OpenAI and pinecone. Then, it sets up the example data, + which in this case are the Agreements we provide as part of our sample data. Next, it creates a library with the + content of the Agreements before it embedds the content. Finally, it performs a sematnic query on the library. + ''' + set_up_api_keys() + + path_agreements = set_up_agreements() + + library = set_up_library(input_folder_path=path_agreements) + + library = set_up_embeddings(library=library) + + query_library(library=library) + + +if __name__ == '__main__': + main() diff --git a/solutions/embeddings/using_qdrant.py b/solutions/embeddings/using_qdrant.py new file mode 100644 index 0000000..7973d5e --- /dev/null +++ b/solutions/embeddings/using_qdrant.py @@ -0,0 +1,97 @@ + +"""This example shows how to use Qdrant as a vector embedding database with llmware + + (A) Python Dependencies - + + As a first step, you should pip install dependencies not included in the llmware package: + -- pip3 install qdrant-client + + (B) Installing Qdrant - + -- Docker - https://qdrant.tech/documentation/guides/installation/ + + (C) Configurations - + + -- set os.environ variables to 'automatically' pass in installing embedding + -- os.environ["USER_MANAGED_QDRANT_HOST"] = "localhost" + -- os.environ["USER_MANAGED_QDRANT_PORT"] = 6333 + +""" + + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + + +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +# start script + +LLMWareConfig().set_active_db("sqlite") + +print("update: Step 1- starting here- building library- parsing PDFs into text chunks") + +lib = build_lib("qdrant_0") + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- before embedding process - check library card - ", lib_card) + +print("update: Step 2 - starting to install embeddings") + +# alt embedding models - "mini-lm-sbert" | industry-bert-contracts | text-embedding-ada-002 +# note: if you want to use text-embedding-ada-002, you will need an OpenAI key and enter into os.environ variable +# e.g., os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + +# batch sizes from 100-500 usually give good performance and work on most environments +lib.install_new_embedding(embedding_model_name="industry-bert-contracts",vector_db="qdrant",batch_size=300) + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- after embedding process - check updated library card - ", lib_card) + +# run a query +# note: embedding_model_name is optional, but useful if you create multiple embeddings on the same library +# --see other example scripts for multiple embeddings + +# create query object +query_pgv = Query(lib, embedding_model_name="industry-bert-contracts") + +# run multiple queries using query_pgv +my_search_results = query_pgv.semantic_query("What is the sale bonus?", result_count = 24) + +for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + +# if you want to delete the embedding - uncomment the line below +# lib.delete_installed_embedding("industry-bert-contracts", "pg_vector") + +# optional - check the embeddings on the library +emb_record = lib.get_embedding_status() +for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + diff --git a/solutions/embeddings/using_redis.py b/solutions/embeddings/using_redis.py new file mode 100644 index 0000000..c5ca1c1 --- /dev/null +++ b/solutions/embeddings/using_redis.py @@ -0,0 +1,100 @@ + +"""This example shows how to use Redis as a vector embedding database with llmware + + (A) Python Dependencies - + + As a first step, you should pip install dependencies not included in the llmware package: + -- pip3 install redis + + (B) Installing Redis - + + If you need help installing Redis, please see the official redis implementation docs (or many widely available tutorials), e.g.,: + -- https://redis.io/docs/install/install-redis/ + -- for a fast development install with docker-compose: + -- please see docker-compose-redis-stack.yaml in the llmware repository + + (C) Configurations - + -- set os.environ variables to 'automatically' pass in installing embedding + -- os.environ["USER_MANAGED_REDIS_HOST"] = "localhost" + -- os.environ["USER_MANAGED_REDIS_PORT"] = 6379 + +""" + + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + + +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +# start script + +LLMWareConfig().set_active_db("sqlite") + +print("update: Step 1- starting here- building library- parsing PDFs into text chunks") + +lib = build_lib("redis_lib_1") + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- before embedding process - check library card - ", lib_card) + +print("update: Step 2 - starting to install embeddings") + +# alt embedding models - "mini-lm-sbert" | industry-bert-contracts | text-embedding-ada-002 +# note: if you want to use text-embedding-ada-002, you will need an OpenAI key and enter into os.environ variable +# e.g., os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + +# batch sizes from 100-500 usually give good performance and work on most environments +lib.install_new_embedding(embedding_model_name="industry-bert-contracts",vector_db="redis",batch_size=300) + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- after embedding process - check updated library card - ", lib_card) + +# run a query +# note: embedding_model_name is optional, but useful if you create multiple embeddings on the same library +# --see other example scripts for multiple embeddings + +# create query object +query_pgv = Query(lib, embedding_model_name="industry-bert-contracts") + +# run multiple queries using query_pgv +my_search_results = query_pgv.semantic_query("What is the sale bonus?", result_count = 24) + +for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + +# if you want to delete the embedding - uncomment the line below +# lib.delete_installed_embedding("industry-bert-contracts", "redis") + +# optional - check the embeddings on the library +emb_record = lib.get_embedding_status() +for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + diff --git a/solutions/embeddings/using_semantic_reranker_with_rag.py b/solutions/embeddings/using_semantic_reranker_with_rag.py new file mode 100644 index 0000000..42a1db9 --- /dev/null +++ b/solutions/embeddings/using_semantic_reranker_with_rag.py @@ -0,0 +1,78 @@ + +""" This example illustrates the use of a reranker model to provide a fast, effective 'in-memory' semantic +similarity, replacing an explicit retrieval using either text or embedding from a vector db. + + This example uses a common pattern of trying to answer a specific factual question from a set of PDF documents. + + These are the key steps of the example: + + 1. Sample agreements are pulled from repo, cached locally, and then iterated. + + 2. Each PDF document (~10-15 pages) is parsed into text chunks in memory. + -- this generates a list of dictionaries, with each dictionary entry consisting of the "text" and metadata + + 3. The reranker model takes as inference input both the query and the full set of text chunks. + + 4. The reranker model returns as output a sorted list of the (top) original text chunk dictionaries + + 5. The top text chunks are added as source to the prompt + + 6. Prompt_with_Sources run with the original query and the concatenated source to answer the question. + + +""" + +import os + +from llmware.parsers import Parser +from llmware.models import ModelCatalog +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def rag_in_memory_with_reranker(): + + """ Executes a rag process in memory using semantic reranker model and bling-phi-3-gguf to answer the question. """ + + query = "What is the annual rate of the executive's base salary?" + + sample_files_path = Setup().load_sample_files(over_write=False) + contracts_path = os.path.join(sample_files_path, "Agreements") + + files = os.listdir(contracts_path) + + # will use two models for the example - reranker + a 'question-answer' rag llm + reranker_model = ModelCatalog().load_model("jina-reranker-turbo") + prompter = Prompt().load_model("bling-phi-3-gguf", temperature=0.0, sample=False) + + for i, doc in enumerate(files): + + if doc.endswith(".pdf"): + + print("\nPROCESSING: ", i, doc) + + parser_output = Parser().parse_one(contracts_path,doc,save_history=False) + + output = reranker_model.inference(query,parser_output,top_n=10, relevance_threshold=0.25) + + use_top = 3 + if len(output) > use_top: + output = output[0:use_top] + + for i, results in enumerate(output): + print("semantic ranking results: ", i, results["rerank_score"], results["text"]) + + sources = prompter.add_source_query_results(output) + responses = prompter.prompt_with_source(query,prompt_name="default_with_context") + + for i, resp in enumerate(responses): + print("\nllm answers: ", i, resp) + + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + rag_in_memory_with_reranker() diff --git a/solutions/embeddings/using_sentence_transformer.py b/solutions/embeddings/using_sentence_transformer.py new file mode 100644 index 0000000..b1f9a0d --- /dev/null +++ b/solutions/embeddings/using_sentence_transformer.py @@ -0,0 +1,113 @@ + +"""This example shows how to use sentence transformers as a vector embedding model with llmware. + +To use models from the SentenceTransformer catalog, you may need to install as follows: + + pip3 install sentence-transformers + +""" + + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.retrieval import Query +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + +from importlib import util +if not util.find_spec("sentence_transformers"): + print("\nto run this example, you should install the SentenceTransformer library with: " + "pip3 install sentence-transformers.") + +def build_lib (library_name, folder="Agreements"): + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # options: Agreements | UN-Resolutions-500 + library.add_files(input_folder_path=os.path.join(sample_files_path, folder)) + + return library + + +# start script + +print("update: Step 1- starting here- building library- parsing PDFs into text chunks") + +LLMWareConfig().set_active_db("sqlite") + +lib = build_lib("st_embedding_0") + +# register a model from the sentence transformers library/repository + +# note: "all-MiniLM-L6-v2" is from the SentenceTransformer catalog, e.g., +# -- https://www.sbert.net/docs/pretrained_models.html +# -- key inputs to register: +# -- "model_name" - should be an existing pre-trained model in the SentenceTransformer catalog +# -- "embedding_dims" - this is the output dimensions, included in the sbert model card info +# -- "context_window" - included in the sbert model card info +# -- *** "model_location" - "st_repo" is reserved word to tell llmware to look in sentence transformers *** +# -- *** "model_family" - "LLMWareSemanticModel" - knows how to load and embed with sentence transformers *** + +# another sentence transformer to try: "all-mpnet-base-v2" - embedding_dims = 768 - context_window = 384 + +sentence_transformer_pretrained_model_name = "all-MiniLM-L6-v2" +embedding_dims = 384 +context_window = 256 + +ModelCatalog().register_sentence_transformer_model(model_name=sentence_transformer_pretrained_model_name, + embedding_dims=embedding_dims, context_window=context_window) + +""" +ModelCatalog().add_model_list({"model_name": sentence_transformer_pretrained_model_name, + "embedding_dims":embedding_dims, + "context_window":context_window, + "model_category": "embedding", + "model_family": "LLMWareSemanticModel", + "display_name": "MySentenceTransformer", "model_location": "st_repo"}) +""" + +# to confirm that model has been added to the catalog +mc = ModelCatalog().list_all_models() +model_card = ModelCatalog().lookup_model_card(sentence_transformer_pretrained_model_name) +print("update: model card - ", model_card) + +# use directly now as an embedding model +lib.install_new_embedding(embedding_model_name=sentence_transformer_pretrained_model_name, + vector_db="milvus",batch_size=300) + +# optional - check the status of the library card and embedding +lib_card = lib.get_library_card() +print("update: -- after embedding process - check updated library card - ", lib_card) + +# create query object (note: including embedding_model is optional - only needed if multiple embeddings on library) +query_st = Query(lib, embedding_model_name=sentence_transformer_pretrained_model_name) + +# run multiple queries using query_pgv +my_search_results = query_st.semantic_query("What is the sale bonus?", result_count = 24) + +for i, qr in enumerate(my_search_results): + print("update: semantic query results: ", i, qr) + +# if you want to delete the embedding - uncomment the line below - including the model_name and vector_db +# lib.delete_installed_embedding(sentence_transformer_pretrained_model_name, "milvus") + +# optional - check the embeddings on the library +emb_record = lib.get_embedding_status() +for j, entries in enumerate(emb_record): + print("update: embeddings on library: ", j, entries) + diff --git a/solutions/gguf/adjusting_sampling_settings.py b/solutions/gguf/adjusting_sampling_settings.py new file mode 100644 index 0000000..16b255c --- /dev/null +++ b/solutions/gguf/adjusting_sampling_settings.py @@ -0,0 +1,52 @@ + +""" This example illustrates how to adjust sampling parameters when loading a model to analyze the impact of + sampling on token selection from the model. + -- note: these parameters are implemented and designed for locally deployed models, e.g., HFGenerativeModel class + and GGUFGenerativeModel class. + -- note: we have seen for function-calling, in particular, that turning sample=False generally yields better + and more consistent results. +""" + +from llmware.models import ModelCatalog + +sample = "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street "\ + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering"\ + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n"\ + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days."\ + "If you have any questions concerning this invoice, contact Bia Hermes. "\ + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000" + + +# the objective of the example is to run several times, and adjust the following parameters to experiment: +# -- sample: True or False +# -- temperature: range between 0.0 - 1.0 (for GGUF models, you can also try setting to negative) +# -- using get_logits and max_output configuration variables + + +# load model and configure sampling parameters +model = ModelCatalog().load_model("bling-stablelm-3b-tool", + sample=False, + temperature=0.0, + get_logits=True, + max_output=123) + +# run a basic summary inference +response = model.inference("What is a list of the key points?", sample) + +# analyze the sampling +sampling_analysis = ModelCatalog().analyze_sampling(response) + +# display the response +print("response: ", response) + +# display the logits +print("logits: ", response["logits"]) + +# show the sampling analysis +print("sampling analysis: ", sampling_analysis) + +# optional (for more detail) - look 'token-by-token' at 'not_top_tokens' selected due to sampling impact +for i, entries in enumerate(sampling_analysis["not_top_tokens"]): + print("sampled choices: ", i, entries) + + diff --git a/solutions/gguf/bling_fast_start.py b/solutions/gguf/bling_fast_start.py new file mode 100644 index 0000000..668cdca --- /dev/null +++ b/solutions/gguf/bling_fast_start.py @@ -0,0 +1,417 @@ + +""" This example demonstrates prompting local BLING models with provided context - easy to select among different +BLING models between 1B - 4B, including both Pytorch versions and GGUF quantized versions, and to swap out the +hello_world questions with your own test set. + + NOTE: if you are running on a CPU with limited memory (e.g., <16 GB of RAM), we would recommend sticking to +the 1B parameter models, or using the quantized GGUF versions. You may get out-of-memory errors and/or very +slow performance with ~3B parameter Pytorch models. Even with 16 GB+ of RAM, the 3B Pytorch models should run but +will be slow (without GPU acceleration). """ + + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +def bling_meets_llmware_hello_world (model_name): + + """ Simple inference loop that loads a model and runs through a series of test questions. """ + + t0 = time.time() + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"] + , prompt_name="default_with_context",temperature=0.30) + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # list of 'rag-instruct' laptop-ready bling models on HuggingFace + + model_list = ["llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3", + + # use GGUF models too + "bling-phi-3-gguf", # quantized bling-phi-3 + "bling-answer-tool", # quantized bling-tiny-llama + "bling-stablelm-3b-tool" # quantized bling-stablelm-3b + ] + + # try the newest bling model - 'tiny-llama' + bling_meets_llmware_hello_world(model_list[1]) + diff --git a/solutions/gguf/chat_models_gguf_fast_start.py b/solutions/gguf/chat_models_gguf_fast_start.py new file mode 100644 index 0000000..00f91b6 --- /dev/null +++ b/solutions/gguf/chat_models_gguf_fast_start.py @@ -0,0 +1,103 @@ +"""This example demonstrates several leading open source chat models running in 4-bit GGUF on local laptop.""" + +import time +import re +from llmware.prompts import Prompt +import logging + + +# Run the benchmark test +def run_test(model_name: str, prompt_list: list[dict]) -> int: + """Run the benchmark test on the specified model with the given prompts. + + Args: + model_name (str): The name of the model to load. + prompt_list (list[dict]): A list of prompts to test the model with. + + Returns: + int: Status code (0 for success). + """ + logging.basicConfig(level=logging.INFO) + + logging.info(f"Loading model '{model_name}'") + + try: + prompter = Prompt().load_model(model_name) + except Exception as e: + logging.error(f"Failed to load model: {e}") + return 1 + + for i, entry in enumerate(prompt_list): + start_time = time.time() + logging.info(f"query - {i+1} - {entry['query']}") + + try: + response = prompter.prompt_main(entry["query"]) + except Exception as e: + logging.error(f"Error during prompting: {e}") + continue + + # Print results + time_taken = round(time.time() - start_time, 2) + llm_response = re.sub("[\n\n]", "\n", response['llm_response']) + logging.info(f"llm_response - {i+1} - {llm_response}") + logging.info(f"time_taken - {i+1} - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + # Example open-ended queries with no context - looking for chat model to draw on general know-how and ability + # to look at a problem conceptually, with focus on language understanding without specific focus on facts + + ds = [ + {"query": "I am interested in gaining an understanding of the banking industry. What topics should I research?", + "context": "", "answer": "NO_GOLD_ANSWER"}, + {"query": "What are some tips for creating a successful business plan?", "context": "", "answer": ""}, + {"query": "What do you think about the recent news about Microsoft\u2019s GitHub acquisition?", "context": "", + "answer": ""}, + {"query": "What are the best books to read for a class on American literature?", "context": "", "answer": ""}, + {"query" : "What is the most important thing I should know about my local school district?", "context": "", "answer": ""}, + {"query": "Can you recommend some good books for me to read?", "context": "", "answer": ""}, + {"query": "What are the differences between the four principal cloud computing service models: IaaS, PaaS, SaaS, and CaaS?", + "context": "", "answer": ""}, + {"query": "I've heard a lot of good things about the iPhone. Should I buy one?", "context": "", "answer": ""}, + {"query": "What is the difference between a sociological and psychological approach to social problems?", + "context": "", "answer": ""}, + {"query": "What job opportunities are available for someone with degree in chemistry?", "context": "", "answer": ""}, + {"query": "I'd like to know how to get the most out of my money in the stock market?", "context": "", "answer": ""}, + {"query": "How do I know if my computer is secure?", "context": "", "answer": ""}, + {"query": "I'm writing an essay on the importance of reading. What are some good questions to ask " + "in this type of essay?", "context": "", "answer": ""}, + {"query": "What is the best way for small businesses to raise capital for their operations?", "context": "", "answer": ""}, + {"query": "I want to start a blog but don't know what to write about. What should I do?", "context": "", "answer": ""}, + {"query": "What are the best ways to learn a new language?", "context": "", "answer": ""}, + {"query": "I'm having some problems with my computer. What are some of the most common computer problems" + " and how can I fix them?", "context": "", "answer": ""}, + {"query": "How can I improve my credit score?", "context": "", "answer": ""}, + {"query": "How can I build confidence in my ability to write articles and be a good writer?", + "context": "", "answer": ""}, + {"query": "How do I negotiate a salary raise?", "context": "", "answer": ""}, + {"query": "What's the best way to learn how to write a good essay?", "context": "", "answer": ""}, + {"query": "I'm a little nervous about taking my first trip abroad. What do I need to know?", "context": "", + "answer": ""}, + {"query": "What is the difference between a dividend and a capital gain?", "context": "", "answer": ""}, + {"query": "What are the best ways to make a house a home?", "context": "", "answer": ""}, + {"query": "I have a question about ecology. What is the difference between a population and a community?", + "context": "", "answer": ""} + ] + + # please note that these models will produce multiple bulletpoints and paragraph length answers, so + # it can take ~30 seconds for each response on a typical Mac M1 laptop + + # for full citations and links, please go to www.huggingface.co/llmware/bonchon model repository + # -- TheBloke/OpenHermes-2.5-Mistral-7B-GGUF + # -- TheBloke/zephyr-7B-beta-GGUF + # -- TheBloke/Starling-LM-7B-alpha-GGUF + # -- TheBloke/Llama-2-7B-Chat-GGUF + # + # -- Llama-3: bartowski/Meta-Llama-3-8B-Instruct-GGUF + # -- Phi-3: microsoft/Phi-3-mini-4k-instruct-gguf + + output = run_test("bartowski/Meta-Llama-3-8B-Instruct-GGUF",ds) diff --git a/solutions/gguf/cohere_command_r_example.py b/solutions/gguf/cohere_command_r_example.py new file mode 100644 index 0000000..0f56b2b --- /dev/null +++ b/solutions/gguf/cohere_command_r_example.py @@ -0,0 +1,24 @@ +""" Example script for using the Cohere Command R model. """ + +from llmware.models import ModelCatalog + +def main(): + """ Demonstrates loading and using the Cohere Command R model. """ + + model_name = "command-r" + + # Load the model using ModelCatalog + model = ModelCatalog().load_model(model_name) + + # Define a prompt for inference + prompt = "Explain the theory of relativity in simple terms." + + # Perform inference + response = model.inference(prompt) + + # Print the response + print("LLM Response:", response["llm_response"]) + print("Usage:", response["usage"]) + +if __name__ == "__main__": + main() diff --git a/solutions/gguf/dragon_gguf_fast_start.py b/solutions/gguf/dragon_gguf_fast_start.py new file mode 100644 index 0000000..f00b4d6 --- /dev/null +++ b/solutions/gguf/dragon_gguf_fast_start.py @@ -0,0 +1,75 @@ + +"""This example demonstrates running a 7B RAG-instruct fine-tuned DRAGON model locally on a laptop. + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + + +import time +from llmware.prompts import Prompt +from llmware.exceptions import LLMWareException +from importlib import util +if not util.find_spec("datasets"): + raise LLMWareException(message="\nto run this example, you need to install HuggingFace datasets: " + "`pip3 install datasets`") + +try: + from datasets import load_dataset +except: + raise LLMWareException(message="Exception: datasets not found and required for example.") + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +# Run the benchmark test +def run_test(model_name, prompt_list): + print(f"\n > Loading model '{model_name}'") + prompter = Prompt().load_model(model_name) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + for i, entry in enumerate(prompt_list): + + start_time = time.time() + + prompt = entry["query"] + context = entry["context"] + response = prompter.prompt_main(prompt, context=context, prompt_name="default_with_context", temperature=0.3) + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i + 1}. llm_response - {response['llm_response']}") + print(f"{i + 1}. gold_answer - {entry['answer']}") + print(f"{i + 1}. time_taken - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + ds = load_rag_benchmark_tester_dataset() + + # Supported Q4_K_M GGUF Dragon Models: + # -- llmware/dragon-yi-6b-gguf + # -- llmware/dragon-mistral-7b-gguf + # -- llmware/dragon-llama-7b-gguf + + model_name = "llmware/dragon-yi-6b-gguf" + + output = run_test(model_name,ds) diff --git a/solutions/gguf/dragon_rag_benchmark_tests_huggingface.py b/solutions/gguf/dragon_rag_benchmark_tests_huggingface.py new file mode 100644 index 0000000..47d989e --- /dev/null +++ b/solutions/gguf/dragon_rag_benchmark_tests_huggingface.py @@ -0,0 +1,116 @@ + +""" This example demonstrates running a benchmarks set of tests against llmware DRAGON models + https://huggingface.co/collections/llmware/dragon-models-65552d7648093c3f6e35d1bf + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + +import time +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError ("This example requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +# Run the benchmark test +def run_test(model_name, test_dataset): + + # Load the model and tokenizer + print(f"\n > Loading model '{model_name}'") + device = "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype="auto") + else: + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True) + model.to(device) + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + # Run each test + for i, entry in enumerate(test_dataset): + + start_time = time.time() + + # Create and tokenize a prompt + # Note: in our testing, the dragon-yi model performs better with a trailing "\n" at end of prompt + new_prompt = ": " + entry["context"] + "\n" + entry["query"] + "\n" + ":" + "\n" + inputs = tokenizer(new_prompt, return_tensors="pt") + start_of_output = len(inputs.input_ids[0]) + + # Call model.generate() + # Note: temperature: set at 0.3 for consistency of output + # max_new_tokens: set at 100 - may prematurely stop a few of the summaries + outputs = model.generate( + inputs.input_ids.to(device), + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.eos_token_id, + do_sample=True, + temperature=0.3, + max_new_tokens=100, + ) + + output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True) + + # quick/optional post-processing clean-up of potential fine-tuning artifacts + + eot = output_only.find("<|endoftext|>") + if eot > -1: + output_only = output_only[:eot] + + bot = output_only.find(":") + if bot > -1: + output_only = output_only[bot+len(":"):] + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i+1}. llm_response - {output_only}") + print(f"{i+1}. gold_answer - {entry['answer']}") + print(f"{i+1}. time_taken - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + # Get the benchmark dataset + test_dataset = load_rag_benchmark_tester_dataset() + + # BLING MODELS + bling_models = ["llmware/bling-1b-0.1", "llmware/bling-1.4b-0.1", "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0"] + + # DRAGON MODELS + dragon_models = ['llmware/dragon-yi-6b-v0', 'llmware/dragon-red-pajama-7b-v0', 'llmware/dragon-stablelm-7b-v0', + 'llmware/dragon-deci-6b-v0', 'llmware/dragon-mistral-7b-v0','llmware/dragon-falcon-7b-v0', + 'llmware/dragon-llama-7b-v0'] + + # Pick a model: if running on CPU/laptop, select from bling_models list + model_name = dragon_models[0] + output = run_test(model_name,test_dataset) diff --git a/solutions/gguf/dragon_rag_benchmark_tests_llmware.py b/solutions/gguf/dragon_rag_benchmark_tests_llmware.py new file mode 100644 index 0000000..7e3eaac --- /dev/null +++ b/solutions/gguf/dragon_rag_benchmark_tests_llmware.py @@ -0,0 +1,98 @@ + +"""This example demonstrates running a benchmarks set of tests against llmware DRAGON models + https://huggingface.co/collections/llmware/dragon-models-65552d7648093c3f6e35d1bf + The model loading and interaction is handled with the llmware Prompt class which provides additional + capabilities like evidence checking + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + +import time +from llmware.prompts import Prompt + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError ("This example requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + +# Run the benchmark test +def run_test(model_name, prompt_list): + + print(f"\n > Loading model '{model_name}'") + prompter = Prompt().load_model(model_name) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + for i, entry in enumerate(prompt_list): + + start_time = time.time() + + prompt = entry["query"] + context = entry["context"] + response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3) + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i+1}. llm_response - {response['llm_response']}") + print(f"{i+1}. gold_answer - {entry['answer']}") + print(f"{i+1}. time_taken - {time_taken}") + + # Fact checking + fc = prompter.evidence_check_numbers(response) + sc = prompter.evidence_comparison_stats(response) + sr = prompter.evidence_check_sources(response) + for fc_entry in fc: + for f, facts in enumerate(fc_entry["fact_check"]): + print(f"{i+1}. fact_check - {f} {facts}") + + for sc_entry in sc: + print(f"{i+1}. comparison_stats - {sc_entry['comparison_stats']}") + + for sr_entry in sr: + for s, source in enumerate(sr_entry["source_review"]): + print(f"{i+1}. source - {s} {source}") + + return 0 + + +if __name__ == "__main__": + + # Get the benchmark dataset + test_dataset = load_rag_benchmark_tester_dataset() + + # BLING MODELS + bling_models = ["llmware/bling-1b-0.1", "llmware/bling-1.4b-0.1", "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0"] + + # DRAGON MODELS + dragon_models = ['llmware/dragon-yi-6b-v0', 'llmware/dragon-red-pajama-7b-v0', 'llmware/dragon-stablelm-7b-v0', + 'llmware/dragon-deci-6b-v0', 'llmware/dragon-mistral-7b-v0','llmware/dragon-falcon-7b-v0', + 'llmware/dragon-llama-7b-v0'] + + # Pick a model - note: if running on laptop/CPU, select a bling model + model_name = dragon_models[0] + output = run_test(model_name, test_dataset) + diff --git a/solutions/gguf/gguf_streaming.py b/solutions/gguf/gguf_streaming.py new file mode 100644 index 0000000..634bc1f --- /dev/null +++ b/solutions/gguf/gguf_streaming.py @@ -0,0 +1,54 @@ + +""" This example illustrates how to use the stream method for GGUF models for fast streaming of inference, +especially for real-time chat interactions. + + Please note that the stream method has been implemented for GGUF models starting in llmware-0.2.13. This will be +any model with GGUFGenerativeModel class, and generally includes models with names that end in "gguf". + + See also the chat UI example in the UI examples folder. + + We would recommend using a chat optimized model, and have included a representative list below. + + +""" + + +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs + +# sets an absolute output maximum for the GGUF engine - normally set by default at 256 +GGUFConfigs().set_config("max_output_tokens", 1000) + +chat_models = ["phi-3-gguf", + "llama-2-7b-chat-gguf", + "llama-3-instruct-bartowski-gguf", + "openhermes-mistral-7b-gguf", + "zephyr-7b-gguf", + "tiny-llama-chat-gguf"] + +model_name = chat_models[0] + +# maximum output can be set optionally at any number up to the "max_output_tokens" set +model = ModelCatalog().load_model(model_name, max_output=500) + +text_out = "" + +token_count = 0 + +# prompt = "I am interested in gaining an understanding of the banking industry. What topics should I research?" +prompt = "What are the benefits of small specialized LLMs?" + +# since model.stream provides a generator, then use as follows to consume the generator + +for streamed_token in model.stream(prompt): + + text_out += streamed_token + if text_out.strip(): + print(streamed_token, end="") + + token_count += 1 + +# final output text and token count + +print("\n\n***total text out***: ", text_out) +print("\n***total tokens***: ", token_count) diff --git a/solutions/gguf/huggingface_integration.py b/solutions/gguf/huggingface_integration.py new file mode 100644 index 0000000..bb5a02d --- /dev/null +++ b/solutions/gguf/huggingface_integration.py @@ -0,0 +1,213 @@ + +""" This example demonstrates the use of HuggingFace models + 1. Use llmware models available on HuggingFace for generating vector embeddings + 2. Load a basic decoder generative model from Huggingface and use it + 3. Customizing a generative model with weights from a custom fine-tuned model + 4. Using a Transformers model for embedding + 5. Using a SentenceTransformers model for embedding +""" + + +import os +import torch +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query +from llmware.models import ModelCatalog, HFEmbeddingModel +from llmware.prompts import Prompt +from llmware.setup import Setup +from llmware.util import CloudBucketManager + + +# note: starting in llmware-0.1.10, transformers and sentence_transformers are included in the pip install + +try: + from transformers import AutoModel, AutoTokenizer, AutoModelForCausalLM +except ImportError: + raise ImportError ( + "This example requires classes from the 'transformers' Python package. " + "You can install it with 'pip install transformers'" + ) +try: + from sentence_transformers import SentenceTransformer +except ImportError: + raise ImportError ( + "This example requires classes from the 'sentence-transformers' Python package" + "You can install it with 'pip install sentence-transformers'" + ) + + +# Load an llmware model from Hugging Face to generate vector embeddings +def use_llmware_hf_models_for_embedding(): + + # llmware industry models currently published on HuggingFace (more will be coming!) + # *** use any HF embedding model, e.g., BERT, Roberta, etc. + # e.g., llmware_industry_models = "llmware/industry-bert-sec-v0.1", + # "llmware/industry-bert-asset-management-v0.1", + # "llmware/industry-bert-contracts-v0.1", + # "llmware/industry-bert-insurance-v0.1" + + # Choose one + hf_model_name = "llmware/industry-bert-sec-v0.1" + + # Load the model using the Transformer classes and then into llmware using an HFEmbeddingModel + print (f"\n > Loading model '{hf_model_name}'from HuggingFace...") + hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_name) + hf_model = AutoModel.from_pretrained(hf_model_name) + + # pass instantiated HF model and tokenizer to HFEmbeddingModel class + llmware_model = HFEmbeddingModel(model=hf_model, tokenizer=hf_tokenizer,model_name=hf_model_name) + + # Generate an vector embedding + sample = "This is a sample sentence" + vector_embedding = llmware_model.embedding(sample) + print (f"\n > Generating a vector embedding for: '{sample}'\n\n{vector_embedding}") + + return vector_embedding + + +# Load a basic decoder generative model from Huggingface and use it +def load_and_use_decoder_generative_model(): + + # These are some good 'off-the-shelf' smaller testing generative models from HuggingFace + hf_model_testing_list = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b", + "EleutherAI/pythia-70m-v0", "EleutherAI/pythia-160m-v0", "EleutherAI/pythia-410m-v0", + "EleutherAI/pythia-1b-v0", "EleutherAI/pythia-1.4b-v0"] + + # Here we'll just select one of the above models + model_name = hf_model_testing_list[6] + + # Load the model using the Transformer classes + print (f"\n > Loading model '{model_name}'from HuggingFace...") + hf_model = AutoModelForCausalLM.from_pretrained(model_name) + hf_tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Bring the model into llware. These models were not trained on instruction following, + # so we set instruction_following to False + model = ModelCatalog().load_hf_generative_model(hf_model, hf_tokenizer, instruction_following=False) + + # Make a call to the model + prompt_text = "The future of artificial intelligence is likely to be" + print (f"\n > Prompting the model with '{prompt_text}'") + output = model.inference(prompt_text)["llm_response"] + print(f"\nResponse:\n{prompt_text}{output}") + + return output + + +# Load a HuggingFace generative model and override the weights to use a custom user-developed fine-tuned model +def override_generative_model_weights_with_custom_fine_tuned_model(): + + # These are some good 'off-the-shelf' smaller testing generative models from HuggingFace + hf_model_testing_list = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b", + "EleutherAI/pythia-70m-v0", "EleutherAI/pythia-160m-v0", "EleutherAI/pythia-410m-v0", + "EleutherAI/pythia-1b-v0", "EleutherAI/pythia-1.4b-v0"] + + # Select a model + model_name = "EleutherAI/pythia-410m-v0" + + # Load the model using the Transformer classes + print (f"\n > Loading model '{model_name}'from HuggingFace...") + hf_model = AutoModelForCausalLM.from_pretrained(model_name) + hf_tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Retrive the custom fine-tuned model + # Note: This is a custom model that has been developed only for testing and demonstration purposes + custom_model_name = "contracts-pythia-hf-410m-v0" + print (f"\n > Loading custom model '{custom_model_name}'from llmware...") + custom_model_path = os.path.join(LLMWareConfig.get_model_repo_path(),custom_model_name) + if not os.path.exists(custom_model_path): + CloudBucketManager().pull_single_model_from_llmware_public_repo(custom_model_name) + + # Override the hf_model default model weights with our own custom-trained weights and load it into llmware + print (f"\n > Overriding model '{model_name}' to use custom-trained weights from '{custom_model_name}'...") + hf_model.load_state_dict(torch.load(os.path.join(custom_model_path,"pytorch_model.bin"), map_location=torch.device('cpu')), strict=False) + model = ModelCatalog().load_hf_generative_model(hf_model, hf_tokenizer, instruction_following=False) + + # Interact with the model + prompt_text = "According to the terms of the executive stock option plan," + print (f"\n > Prompting the model with '{prompt_text}'") + output = model.inference(prompt_text)["llm_response"] + print(f"\nResponse:\n{prompt_text}{output}") + + return output + + +# Use a Transformers model for embedding +def use_transformers_model_for_embedding(library_name, model_name): + + # Create a library and add some documents so we can do some vector embeddings + print (f"\n > Creating a library...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "SmallLibrary")) + + # Load the model + print (f"\n > Loading model '{model_name}'") + hf_model = AutoModel.from_pretrained(model_name) + hf_tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Create vector embeddings + print (f"\n > Creating vector embeddings...") + library.install_new_embedding(model=hf_model, tokenizer=hf_tokenizer, from_hf=True, vector_db="faiss", batch_size=50) + + # Perform a query + query_term = "salary" + print (f"\n > Performing query for {query_term}...") + query = Query(library=library, embedding_model_name=model_name, embedding_model=hf_model, tokenizer=hf_tokenizer, from_hf=True) + query_results = query.semantic_query(query_term,result_count=3) + print (f"Top 3 Results:") + for i, result in enumerate(query_results): + file_source = result["file_source"] + page_num = result["page_num"] + text = result["text"] + print(f"\n - From {file_source} (page {page_num}):\n{text}") + + return 0 + + +# Use a SentenceTransformers model for embedding +def use_sentence_transformers_model_for_embedding(library_name, model_name): + + # Create a library and add some documents so we can do some vector embeddings + print (f"\n > Creating a library...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "SmallLibrary")) + + # Load the model + print (f"\n > Loading model '{model_name}'") + sbert_model = SentenceTransformer(model_name) + + # Create vector embeddings + print (f"\n > Creating vector embeddings...") + library.install_new_embedding(model=sbert_model, embedding_model_name=model_name, + from_sentence_transformer=True, vector_db="faiss", batch_size=100) + + # Perform a query + query_term = "salary" + print (f"\n > Performing query for {query_term}...") + + query= Query(library=library, + embedding_model_name=model_name, + embedding_model=sbert_model, + from_sentence_transformer=True) + + query_results = query.semantic_query(query_term, result_count=3) + print (f"Top 3 Results:") + for i, result in enumerate(query_results): + file_source = result["file_source"] + page_num = result["page_num"] + text = result["text"] + print(f"\n - From {file_source} (page {page_num}):\n{text}") + + return query_results + + +if __name__ == "__main__": + + use_llmware_hf_models_for_embedding() + load_and_use_decoder_generative_model() + override_generative_model_weights_with_custom_fine_tuned_model() + use_transformers_model_for_embedding("test_transformers", "bert-base-cased") + use_sentence_transformers_model_for_embedding("test_sentence_transformers", "all-distilroberta-v1") diff --git a/solutions/gguf/llmware_model_fast_start.py b/solutions/gguf/llmware_model_fast_start.py new file mode 100644 index 0000000..fed57fc --- /dev/null +++ b/solutions/gguf/llmware_model_fast_start.py @@ -0,0 +1,166 @@ + +"""This example demonstrates running a benchmarks set of tests against any llmware model in HuggingFace + https://huggingface.co/llmware + + Usage: You can pass in a model name: + python llmware_model_fast_start.py llmware/bling-1b-0.1 + If you do not specify a model you will be prompted to pick one + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + +import re +import sys +import time +import torch +from huggingface_hub import hf_api, ModelCard +from transformers import AutoModelForCausalLM, AutoTokenizer + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError ("This example requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + + +# Query HuggingFace and get the llmware models. Return the the components of a table: headers and data +def get_llmware_models(): + table_headers=['','MODEL','DETAILS'] + table_data=[] + + models = hf_api.list_models(author="llmware") + sorted_models = sorted(models, key=lambda x: x.id) + + for i, model in enumerate(sorted_models): + model_card_content = ModelCard.load(model.id).content + match = re.search(r"Model type:\*\* (.+?)\n", model_card_content) # Get type from a line like this: - **Model type:** GPTNeoX instruct-trained decoder + model_type = "" + if match: + model_type = match.group(1).strip() + model_details = f"{model_type} ({model.downloads} downloads)" + table_data.append([i+1, model.id, model_details]) + + return table_headers, table_data + + +def print_llmware_models(): + + table_headers, table_data = get_llmware_models() + + print(table_headers[0], "\t\t", table_headers[1], "\t\t", table_headers[2]) + for row in table_data: + print(row[0], "\t\t", row[1], "\t\t", row[2]) + + +def prompt_user_for_model_selection(prompt=None): + + table_headers, table_data = get_llmware_models() + + print(table_headers[0], "\t\t", table_headers[1], "\t\t", table_headers[2]) + for row in table_data: + print(row[0], "\t\t", row[1], "\t\t", row[2]) + + num_models = len(table_data) + + if prompt is None: + prompt = f"\nSelect a model (1-{num_models}): " + while True: + try: + user_input = input(prompt) + user_integer = int(user_input) + if user_integer not in range(1,num_models+1): + continue + return table_data[user_integer-1][1] + except ValueError: + print("That's not an integer. Please try again.") + return None + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +# Run the benchmark test +def run_test(model_name, test_dataset): + + # Load the model and tokenizer + print(f"\n > Loading model '{model_name}'") + device = "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype="auto") + else: + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True) + model.to(device) + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + # Run each test + for i, entry in enumerate(test_dataset): + + start_time = time.time() + + # Create and tokenize a prompt + # Note: in our testing, the dragon-yi model performs better with a trailing "\n" at end of prompt + new_prompt = ": " + entry["context"] + "\n" + entry["query"] + "\n" + ":" + "\n" + inputs = tokenizer(new_prompt, return_tensors="pt") + start_of_output = len(inputs.input_ids[0]) + + # Call model.generate() + # Note: temperature: set at 0.3 for consistency of output + # max_new_tokens: set at 100 - may prematurely stop a few of the summaries + outputs = model.generate( + inputs.input_ids.to(device), + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.eos_token_id, + do_sample=True, + temperature=0.3, + max_new_tokens=100, + ) + + output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True) + + # quick/optional post-processing clean-up of potential fine-tuning artifacts + + eot = output_only.find("<|endoftext|>") + if eot > -1: + output_only = output_only[:eot] + + bot = output_only.find(":") + if bot > -1: + output_only = output_only[bot+len(":"):] + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i+1}. llm_response - {output_only}") + print(f"{i+1}. gold_answer - {entry['answer']}") + print(f"{i+1}. time_taken - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + # Prompt user to get model if not passed in as an argument + if len(sys.argv) > 1: + selected_model = sys.argv[1] + else: + selected_model = prompt_user_for_model_selection() + test_dataset = load_rag_benchmark_tester_dataset() + output = run_test(selected_model,test_dataset) diff --git a/solutions/gguf/model_catalog_registry.py b/solutions/gguf/model_catalog_registry.py new file mode 100644 index 0000000..650ef4a --- /dev/null +++ b/solutions/gguf/model_catalog_registry.py @@ -0,0 +1,55 @@ + +"""This example illustrates the basic capabilities of the Model Catalog and how to use""" + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# 1 - to list all of the models registered by default in llmware +mc = ModelCatalog().list_all_models() + +print("\nAll-Models-List") +for i, models in enumerate(mc): + print("update: models - ", i, models) + + +# 2 - to list all generative local models +gl_mc = ModelCatalog().list_generative_local_models() + +print("\nGenerative-Local-Models-List") +for i, models in enumerate(gl_mc): + print("update: gen local models - ", i, models) + + +# 3 - to add models to the catalog, e.g., sentence transformer + +ModelCatalog().register_sentence_transformer_model("all-MiniLM-L6-v2", embedding_dims=384, + context_window=256) + +""" +ModelRegistry().add_model_list({"model_name": "all-MiniLM-L6-v2", "model_category": "embedding", + "embedding_dims":384, "context_window":256, "model_family": "LLMWareSemanticModel", + "display_name": "MiniLM", "model_location": "st_repo"}) +""" + +# to confirm that model was added +model_card = ModelCatalog().lookup_model_card("all-MiniLM-L6-v2") +print("\nupdate: Registered new embedding model - ", model_card) + +# 4 - embedding models +emb_models = ModelCatalog().list_embedding_models() + +# note: newly created 'all-MiniLM-L6-v2' will now be included +print("\nEmbedding-Models-List") +for i, models in enumerate(emb_models): + print("update: embedding models - ", i, models) + +# 5 - to delete model from catalog +ModelCatalog().delete_model_card("gpt-3.5-turbo") + +# 6 - to load any model in the catalog +new_model = ModelCatalog().load_model("all-MiniLM-L6-v2") + +# 7 - equivalent 'load_model' with any prompt +# -- note: if this model is not installed locally, this will pull it down into local cache +prompter = Prompt().load_model("llmware/bling-1b-0.1") + diff --git a/solutions/gguf/using-azure-openai.py b/solutions/gguf/using-azure-openai.py new file mode 100644 index 0000000..2d31215 --- /dev/null +++ b/solutions/gguf/using-azure-openai.py @@ -0,0 +1,72 @@ + +""" This example shows how to use OpenAIConfigs to create a configured OpenAI client, most often used for +Azure OpenAI access.""" + +import os + +from llmware.models import ModelCatalog +from llmware.configs import OpenAIConfig +from openai import AzureOpenAI + + +# Set the following environment variables: +# - AZURE_OPENAI_ENDPOINT : found on your Azure OpenAI page +# - AZURE_OPENAI_API_KEY : found on your Azure OpenAI page +# - USER_MANAGED_OPENAI_API_KEY : found on you OpenAI API page +# +# Additionally, with this example, you will need an Azure OpenAI deployment +# for gpt-4 and text-embedding-3-small, but feel free to replace these below. +# +# Make sure to replace the deployment names with your deployments in the +# AzureOpenAI clients created below. + + +# to start - OpenAI client is created in OpenAI Generative and Embedding models classes at the time of inference +# the client will be created as a standard OpenAI client with the api_keys passed + +my_azure_client = OpenAIConfig().get_azure_client() +print("my azure client to start: ", my_azure_client) + +# to configure an AzureOpenAI client, two steps: +# first, create the client with openai >= 1.0 python SDK, (see above) e.g.: + +gpt4_client = AzureOpenAI( + azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT"), + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + api_version="2024-02-01", + azure_deployment="your-gpt-4-deployment-name" +) + +# second, set the azure client in OpenAIConfigs as below: +OpenAIConfig().set_azure_client(gpt4_client) +print("my azure client - set: ", OpenAIConfig().get_azure_client()) + +# now, run the inference like any other in llmware + +# OpenAI Generative call +model = ModelCatalog().load_model("gpt-4") + +# the model will check the value of get_azure_client() in the configs -> if set, then will use +response = model.inference("What is the future of AI") +print("response: ", response) + +text_embedding_client = AzureOpenAI( + azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT"), + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + api_version="2024-02-01", + azure_deployment="your-text-embedding-3-small-deployment-name" +) + +OpenAIConfig().set_azure_client(text_embedding_client) + +# OpenAI Embedding call +model = ModelCatalog().load_model("text-embedding-3-small") +embedding = model.embedding(["This is a sample sentence for an embedding test."]) +print("embedding: ", embedding) + +# reset so you can use the standard OpenAI client +OpenAIConfig().set_azure_client(None) + +model = ModelCatalog().load_model("text-embedding-3-small", api_key=os.getenv("USER_MANAGED_OPENAI_API_KEY")) +embedding = model.embedding(["This is a sample sentence for an embedding test."]) +print("embedding: ", embedding) diff --git a/solutions/gguf/using-custom-gguf-model.py b/solutions/gguf/using-custom-gguf-model.py new file mode 100644 index 0000000..f806c21 --- /dev/null +++ b/solutions/gguf/using-custom-gguf-model.py @@ -0,0 +1,51 @@ + +""" This example shows how to use any gguf model available on HuggingFace, and start using in inferences and +workflows with llmware. In this scenario, we will take the following steps: + + 1. Register new GGUF model + 2. Register new finetune wrapper, if needed + 3. Start running inferences +""" + +import time +import re + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# Step 1 - register new gguf model - we will pick the popular LLama-2-13B-chat-GGUF + +ModelCatalog().register_gguf_model(model_name="TheBloke/Llama-2-13B-chat-GGUF-Q2", + gguf_model_repo="TheBloke/Llama-2-13B-chat-GGUF", + gguf_model_file_name="llama-2-13b-chat.Q2_K.gguf", + prompt_wrapper="my_version_inst") + +# Step 2- if the prompt_wrapper is a standard, e.g., Meta's , then no need to do anything else +# -- however, if the model uses a custom prompt wrapper, then we need to define that too +# -- in this case, we are going to create our "own version" of the Meta wrapper + +ModelCatalog().register_new_finetune_wrapper("my_version_inst", main_start="", llm_start="") + +# Once we have completed these two steps, we are done - and can begin to use the model like any other + +prompter = Prompt().load_model("TheBloke/Llama-2-13B-chat-GGUF-Q2") + +question_list = ["I am interested in gaining an understanding of the banking industry. What topics should I research?", + "What are some tips for creating a successful business plan?", + "What are the best books to read for a class on American literature?"] + + +for i, entry in enumerate(question_list): + + start_time = time.time() + print("\n") + print(f"query - {i + 1} - {entry}") + + response = prompter.prompt_main(entry) + + # Print results + time_taken = round(time.time() - start_time, 2) + llm_response = re.sub("[\n\n]", "\n", response['llm_response']) + print(f"llm_response - {i + 1} - {llm_response}") + print(f"time_taken - {i + 1} - {time_taken}") + diff --git a/solutions/gguf/using-llama-3.py b/solutions/gguf/using-llama-3.py new file mode 100644 index 0000000..358d2a0 --- /dev/null +++ b/solutions/gguf/using-llama-3.py @@ -0,0 +1,31 @@ + +""" This example shows how to use the new Llama-3 Model with llmware, as well as how to access quantized versions. """ + +from llmware.models import ModelCatalog + +# *** CORE PYTORCH LLAMA-3 MODELS *** + +# llama-3-8b models pre-registered in the model catalog: +# llama-3-base - "Meta-Llama-3-8B-Instruct" or "llama-3-instruct" +# llama-3-instruct - "Meta-Llama-3-8B" or "llama-3-base" + +# note: to access these models in llmware requires two pre-registration steps: +# 1. meta-llama registration - https://llama.meta.com/docs/get-started/ - requires accepting the llama-3 licensing terms +# 2. huggingface api key (does not require any payment, but you need a free HF account), +# e.g., hf_key = "hf_...." + +# once you have completed these steps, you can access in llmware as follows: +# # llama3_model = ModelCatalog().load_model(selected_llama_model) + +# *** LLAMA-3 GGUF MODELS *** + +# 3 quantized models added to the default Model Catalog: +# model_name = "bartowski/Meta-Llama-3-8B-Instruct-GGUF" +# model_name = "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF" +# model_name = "QuantFactory/Meta-Llama-3-8B-GGUF" + +l3_gguf = ModelCatalog().load_model("bartowski/Meta-Llama-3-8B-Instruct-GGUF") + +response = l3_gguf.inference("I am going to Mumbai. What should I see?") +print("\nllama3-gguf response: ", response) + diff --git a/solutions/gguf/using-microsoft-phi-3.py b/solutions/gguf/using-microsoft-phi-3.py new file mode 100644 index 0000000..557ba89 --- /dev/null +++ b/solutions/gguf/using-microsoft-phi-3.py @@ -0,0 +1,30 @@ + +""" This example shows how to use the new Microsoft Phi-3 model. """ + +from llmware.models import ModelCatalog + +# phi-3 models pre-registered in the model catalog (as of Tues, April 23 when model launched): +# phi-3 - "microsoft/Phi-3-mini-4k-instruct" +# phi-3-128k - "microsoft/Phi-3-mini-128k-instruct" +# phi-3-gguf - "microsoft/Phi-3-mini-4k-instruct-gguf" + +# first let's try the pytorch version +# note: if not running on a cuda machine, you may see warnings about flash_attn not present +# ... and it will be a little slow to load + +phi3 = ModelCatalog().load_model("phi-3") # use "phi-3-128k" for the 128k context +response = phi3.inference("I am going to Mumbai. What should I see?") +print("\nresponse: ", response) + +# second, use the gguf version +phi3_gguf = ModelCatalog().load_model("phi-3-gguf") + +response = phi3_gguf.inference("I am going to Mumbai. What should I see?") +print("\ngguf response: ", response) + +# now, try with a context sample +context = "The stock is now soaring to $120 per share after great earnings." +response = phi3_gguf.inference("What is the current stock price?", add_context=context) + +print("\ngguf response: ", response) + diff --git a/solutions/gguf/using-ollama-models.py b/solutions/gguf/using-ollama-models.py new file mode 100644 index 0000000..d0d4588 --- /dev/null +++ b/solutions/gguf/using-ollama-models.py @@ -0,0 +1,81 @@ + +""" This example illustrates how to use Ollama models in llmware. It assumes that you have separately + downloaded and installed Ollama and used 'ollama run {model_name}' to cache several models in + ollama. """ + +from llmware.models import ModelCatalog + +# Step 1 - register your Ollama models in llmware ModelCatalog +# -- these two lines will register: llama2 and mistral models +# -- note: assumes that you have previously cached and installed both of these models with ollama locally + +# register llama2 +ModelCatalog().register_ollama_model(model_name="llama2",model_type="chat",host="localhost",port=11434) + +# register mistral - note: if you are using ollama defaults, then OK to register with ollama model name only +ModelCatalog().register_ollama_model(model_name="mistral") + +# optional - confirm that model was registered +my_new_model_card = ModelCatalog().lookup_model_card("llama2") +print("\nupdate: confirming - new ollama model card - ", my_new_model_card) + +# Step 2 - start using the Ollama model like any other model in llmware + +print("\nupdate: calling ollama llama 2 model ...") + +model = ModelCatalog().load_model("llama2") +response = model.inference("why is the sky blue?") + +print("update: example #1 - ollama llama 2 response - ", response) + +# Tip: if you are loading 'llama2' chat model from Ollama, note that it is already included in +# the llmware model catalog under a different name, "TheBloke/Llama-2-7B-Chat-GGUF" +# the llmware model name maps to the original HuggingFace repository, and is a nod to "TheBloke" who has +# led the popularization of GGUF - and is responsible for creating most of the GGUF model versions. +# --llmware uses the "Q4_K_M" model by default, while Ollama generally prefers "Q4_0" + +print("\nupdate: calling Llama-2-7B-Chat-GGUF in llmware catalog ...") + +model = ModelCatalog().load_model("TheBloke/Llama-2-7B-Chat-GGUF") +response = model.inference("why is the sky blue?") + +print("update: example #1 - [compare] - llmware / Llama-2-7B-Chat-GGUF response - ", response) + +# Now, let's try the Ollama Mistral model with a context passage + +model2 = ModelCatalog().load_model("mistral") + +context_passage= ("NASA’s rover Perseverance has gathered data confirming the existence of ancient lake " + "sediments deposited by water that once filled a giant basin on Mars called Jerezo Crater, " + "according to a study published on Friday. The findings from ground-penetrating radar " + "observations conducted by the robotic rover substantiate previous orbital imagery and " + "other data leading scientists to theorize that portions of Mars were once covered in water " + "and may have harbored microbial life. The research, led by teams from the University of " + "California at Los Angeles (UCLA) and the University of Oslo, was published in the " + "journal Science Advances. It was based on subsurface scans taken by the car-sized, six-wheeled " + "rover over several months of 2022 as it made its way across the Martian surface from the " + "crater floor onto an adjacent expanse of braided, sedimentary-like features resembling, " + "from orbit, the river deltas found on Earth.") + +response = model2.inference("What are the top 3 points?", add_context=context_passage) + +print("\nupdate: calling ollama mistral model ...") + +print("update: example #2 - ollama mistral response - ", response) + +# Step 3 - using the ollama discovery API - optional + +discovery = model2.discover_models() +print("\nupdate: example #3 - checking ollama model manifest list: ", discovery) + +if len(discovery) > 0: + # note: assumes tht you have at least one model registered in ollama -otherwise, may throw error + for i, models in enumerate(discovery["models"]): + print("ollama models: ", i, models) + + +# for more information and other alternatives for using GGUF models, please see the following examples: +# -- examples/Models/chat_gguf_fast_start.py +# -- examples/Models/using_gguf.py +# -- examples/Models/using-open-chat-models.py +# -- examples/Models/dragon-gguf_fast_start.py diff --git a/solutions/gguf/using-open-chat-models.py b/solutions/gguf/using-open-chat-models.py new file mode 100644 index 0000000..b4b6c75 --- /dev/null +++ b/solutions/gguf/using-open-chat-models.py @@ -0,0 +1,60 @@ + +""" + This example shows how to use 'Open Chat' inference models that expose an endpoint compatible with the + OpenAI API - using 'api_base' to configure the endpoint uri + + For example, to integrate a model on LM Studio with standard configuration: + -- api_base = 'http://localhost:1234/v1' + + Please also note that llmware implements llama.cpp directly, so you can run inference on any GGUF models + very easily and natively in llmware - see the GGUF example in /Models/using_gguf.py' +""" + + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + + +# one step process: add the open chat model to the Model Registry +# key params: +# model_name = "my_open_chat_model1" +# api_base = uri_path to the proposed endpoint +# prompt_wrapper = alpaca | | chat_ml | hf_chat | human_bot +# -> Llama2-Chat +# hf_chat -> Zephyr-Mistral +# chat_ml -> OpenHermes - Mistral +# human_bot -> Dragon models +# model_type = "chat" (alternative: "completion") + +ModelCatalog().register_open_chat_model("my_open_chat_model1", + api_base="http://localhost:1234/v1", + prompt_wrapper="", + model_type="chat") + +# once registered, you can invoke like any other model in llmware + +prompter = Prompt().load_model("my_open_chat_model1") +response = prompter.prompt_main("What is the future of AI?") + + +# you can (optionally) register multiple open chat models with different api_base and model attributes + +ModelCatalog().register_open_chat_model("my_open_chat_model2", + api_base="http://localhost:5678/v1", + prompt_wrapper="hf_chat", + model_type="chat") + + +# you can also alternate with open ai models - which will 'revert' to the default openai api_base + +openai_prompter = Prompt().load_model("gpt-3.5.-turbo-instruct") + + +# if you list all of the models in the catalog, you will see the two newly created open chat models + +my_models = ModelCatalog().list_all_models() + +for i, mods in enumerate(my_models): + print("models: ", i, mods) + + diff --git a/solutions/gguf/using-phi-3-function-calls.py b/solutions/gguf/using-phi-3-function-calls.py new file mode 100644 index 0000000..f7f100c --- /dev/null +++ b/solutions/gguf/using-phi-3-function-calls.py @@ -0,0 +1,99 @@ + +""" This example shows how to use 7 different SLIM function calling models fine-tuned on top of Phi-3: + + -- Extraction - slim-extract-phi-3-gguf - generates python dictionary with 'key' and 'value' + -- Summarization - slim-summary-phi-3-gguf - generates python list with key bullet-point summary + -- XSUM (titles) - slim-xsum-phi-3-gguf - generates python dictionary with 'xsum' key + -- Boolean - slim-boolean-phi-3-gguf - generate python dictionary with "answer" & "explanation" keys + -- Sentiment-NER - slim-sa-ner-phi-3-gguf - generates python dictionary with "sentiment" and selected ner keys + -- Question-Gen - slim-q-gen-phi-3-tool - generates python dictionary with "question" key + -- Question-Answer - slim-qa-gen-phi-3-tool - generates python dictionary with "question" and "answer" key + + The design of these models is to simplify both the input prompt and output to enable easy integration into + programmatic workflows. + + """ + +from llmware.models import ModelCatalog + +# sample text passage that will be used as the basis for the function call analysis + +context_passage = ("Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter.") + + +# for convenience to execute a 'loop', we will set up a dictionary with each function call, and the associated +# model and parameters that are being passed to the model + +phi3_function_call_models = { + + # extract model will look for the 'key' in the params, and return the 'value' found in the text + "extract": {"model": "slim-extract-phi-3-gguf", "params": ["net income"]}, + + # summary model will return a python list with key summary points related to the parameter + "summary": {"model": "slim-summary-phi-3-gguf", "params": ["financial highlights"]}, + + # xsum model produces an 'extreme summarization', e.g. a headline or title + "xsum": {"model": "slim-xsum-phi-3-gguf", "params": ["xsum"]}, + + # boolean model is designed to answer yes/no questions + "boolean": {"model": "slim-boolean-phi-3-gguf", "params": ["Is Best Buy closing stores? (explain)"]}, + + # sentiment-ner model returns several keys for sentiment and ner attributes (e.g., people, place, organization) + "sentiment-ner": {"model": "slim-sa-ner-phi-3-gguf", "params": ["sentiment", "people"]}, + + # q-gen model generates a question from the context passage + "q-gen": {"model": "slim-q-gen-phi-3-tool", "params": ["question"]}, + + # qa-gen model generates question and answer from the context passage + "qa-gen": {"model": "slim-qa-gen-phi-3-tool", "params": ["question, answer"]} + } + +for function, model in phi3_function_call_models.items(): + + print(f"\nfunction: {function} - model - {model['model']} - params - {model['params']}") + slim_model = ModelCatalog().load_model(model["model"], temperature=0.0, sample=False) + + # note: this is the line doing all of the work - each model has been fine-tuned as a 'specialist' for + # its function, so the only required inputs are the source context passage, and the specific parameters to be used + + response = slim_model.function_call(context_passage, params=model["params"]) + + print("response: ", response) + + + diff --git a/solutions/gguf/using-qwen2-models.py b/solutions/gguf/using-qwen2-models.py new file mode 100644 index 0000000..c3c2334 --- /dev/null +++ b/solutions/gguf/using-qwen2-models.py @@ -0,0 +1,57 @@ + +""" This example shows how to use Qwen2 models in LLMWare, consisting of three main categories - + + 1 - standard QWEN2 chat/instruct models, packaged in GGUF in 7B / 1.5B / 0.5B sizes. + + 2 - RAG fine-tuned QWEN2 in DRAGON and BLING series. + + 3 - Extract function-calling finetune in SLIM series. + +""" + +from llmware.models import ModelCatalog + +# 1 - MAIN CATALOG - 3 QWEN2 GGUF models for chat (7B / 1.5B / 0.5B) + +qwen2_base_gguf = ["qwen2-7b-instruct-gguf", "qwen2-1.5b-instruct-gguf", "qwen2-0.5b-instruct-gguf"] + +print("\nExample #1 - loading Qwen2-instruct model - may take a minute the first time.") + +qwen2 = ModelCatalog().load_model("qwen2-1.5b-instruct-gguf", max_output=200) +response = qwen2.inference("I am going to visit Istanbul. What should I see?") +print("\nresponse: ", response) + +# 2 - RAG FINETUNE - DRAGON + BLING + +print("\nExample #2 - RAG finetuned Qwen2 for fact-based question answering with context passage.") + +qwen2_rag_finetunes = ["dragon-qwen-7b-gguf", "bling-qwen-1.5b-gguf", "bling-qwen-0.5b-gguf"] + +qwen2_rag = ModelCatalog().load_model("bling-qwen-1.5b-gguf", temperature=0.0, sample=False) +context = "The stock is now soaring to $120 per share after great earnings." +response = qwen2_rag.inference("What is the current stock price?", add_context=context) + +print("\nqwen2-rag response: ", response) + + +# 3 - FUNCTION-CALLING EXTRACTION SLIM MODELS + +print("\nExample #3 - Qwen2 Extract function calling model.") + +qwen2_extract_function_calls = ["slim-extract-qwen-1.5b-gguf", "slim-extract-qwen-0.5b-gguf"] + +context_passage = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + +qwen2_extract = ModelCatalog().load_model("slim-extract-qwen-1.5b-gguf",temperature=0.0,sample=False) +response = qwen2_extract.function_call(context_passage, params=["earnings per share"]) + +print("\nqwen2-extract response: ", response) + diff --git a/solutions/gguf/using-whisper-cpp-getting-started.py b/solutions/gguf/using-whisper-cpp-getting-started.py new file mode 100644 index 0000000..0f55a78 --- /dev/null +++ b/solutions/gguf/using-whisper-cpp-getting-started.py @@ -0,0 +1,86 @@ + +""" This example shows how to get started using WhisperCPP as a fast, local voice-to-text processing engine. + + Whisper is a leading open voice voice-to-text model from OpenAI - https://github.com/openai/whisper + + WhisperCPP is the implementation of Whisper packaged as a GGML deliverable - https://github.com/ggerganov/whisper.cpp + + Starting with llmware 0.2.11, we have integrated WhisperCPPModel as a new model class, + providing options for direct inference, and coming soon, integration into the Parser for easy text chunking and + parsing into a Library with other document types. + + llmware provides prebuilt shared libraries for WhisperCPP on the following platforms: + --Mac M series + --Linux x86 (no CUDA) + --Linux x86 (with CUDA) - really fast + --Windows x86 (only on CPU) currently. + + We have added three Whisper models to the default model catalog: + 1. ggml-base.en.bin - english-only base model + 2. ggml-base.bin - multi-lingual base model + 2. ggml-small.en-tdrz.bin - this is a 'tiny-diarize' implementation that has been finetuned to identify the + speakers and inserts special [_SOLM_] tags to indicate a conversation turn / change of speaker. + Main repo: https://github.com/akashmjn/tinydiarize/ + Citation: @software{mahajan2023tinydiarize, + author = {Mahajan, Akash}, month = {08}, + title = {tinydiarize: Minimal extension of Whisper for speaker segmentation with special tokens}, + url = {https://github.com/akashmjn/tinydiarize}, + year = {2023} + + To use WAV files, there is one additional Python dependency required: + --pip install librosa + --Note: this has been added to the default requirements.txt and pypy build starting with 0.2.11 + + To use other popular audio/video file formats, such as MP3, MP4, M4A, etc., then the following dependencies are + required: + --pip install pydub + --ffmpeg library - which can be installed as follows: + -- Linux: `sudo apt install ffmpeg' + -- Mac: `brew install ffmpeg` + -- Windows: direct download and install from ffmpeg + + """ + +import os +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs + +# optional / to adjust various log/display parameters of the model +GGUFConfigs().set_config("whisper_cpp_verbose", "OFF") +GGUFConfigs().set_config("whisper_cpp_realtime_display", True) + +# note: english is default output - change to 'es' | 'fr' | 'de' | 'it' ... +GGUFConfigs().set_config("whisper_language", "en") + +# whether to add or remove segment markers in llm response output +GGUFConfigs().set_config("whisper_remove_segment_markers", True) + + +def basic_whisper_cpp_use_example(): + + """ Hello world example to get started using WhisperCPP in LLMWare. """ + + fp = "/local/path/to/.wav" + fn = "my_wav.wav" + + # prompt = string representing the path to a .wav file + prompt = os.path.join(fp,fn) + + # choose between english-only and multilingual + whisper_base_english = "whisper-cpp-base-english" + whisper_base_multi = "whisper-cpp-base" + + # load and run inference like any other model in llmware + model = ModelCatalog().load_model(whisper_base_english) + response = model.inference(prompt) + + print("\nllm response: ", response["llm_response"]) + print("usage: ", response["usage"]) + + return response + + +if __name__ == "__main__": + + response = basic_whisper_cpp_use_example() + diff --git a/solutions/gguf/using-whisper-cpp-sample-files.py b/solutions/gguf/using-whisper-cpp-sample-files.py new file mode 100644 index 0000000..09939bf --- /dev/null +++ b/solutions/gguf/using-whisper-cpp-sample-files.py @@ -0,0 +1,81 @@ + +""" This example shows how to use llmware provided sample files for testing with WhisperCPP, integrated as of + llmware 0.2.11. + + # examples - "famous_quotes" | "greatest_speeches" | "youtube_demos" | "earnings_calls" + + -- famous_quotes - approximately 20 small .wav files with clips from old movies and speeches + -- greatest_speeches - approximately 60 famous historical speeches in english + -- youtube_videos - wav files of ~3 llmware youtube videos + -- earnings_calls - wav files of ~4 public company earnings calls (gathered from public investor relations) + + These sample files are hosted in a non-restricted AWS S3 bucket, and downloaded via the Setup method + `load_sample_voice_files`. There are two options: + + -- small_only = True: only pulls the 'famous_quotes' samples + -- small_only = False: pulls all of the samples (requires ~1.9 GB in total) + + Please note that all of these samples have been pulled from open public domain sources, including the + Internet Archives, e.g., https://archive.org. These sample files are being provided solely for the purpose of + testing the code scripts below. Please do not use them for any other purpose. + + To run these examples, please make sure to `pip install librosa` + """ + +import os +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs +from llmware.setup import Setup + +# optional / to adjust various parameters of the model +GGUFConfigs().set_config("whisper_cpp_verbose", "OFF") +GGUFConfigs().set_config("whisper_cpp_realtime_display", True) + +# note: english is default output - change to 'es' | 'fr' | 'de' | 'it' ... +GGUFConfigs().set_config("whisper_language", "en") +GGUFConfigs().set_config("whisper_remove_segment_markers", True) + + +def sample_files(example="famous_quotes", small_only=False): + + """ Execute a basic inference on Voice-to-Text model passing a file_path string """ + + voice_samples = Setup().load_voice_sample_files(small_only=small_only) + + examples = ["famous_quotes", "greatest_speeches", "youtube_demos", "earnings_calls"] + + if example not in examples: + print("choose one of the following - ", examples) + return 0 + + fp = os.path.join(voice_samples,example) + + files = os.listdir(fp) + + # these are the two key lines + whisper_base_english = "whisper-cpp-base-english" + + model = ModelCatalog().load_model(whisper_base_english) + + for f in files: + + if f.endswith(".wav"): + + prompt = os.path.join(fp,f) + + print(f"\n\nPROCESSING: prompt = {prompt}") + + response = model.inference(prompt) + + print("\nllm response: ", response["llm_response"]) + print("usage: ", response["usage"]) + + return 0 + + +if __name__ == "__main__": + + # pick among the four examples: famous_quotes | greatest_speeches | youtube_demos | earnings_calls + + sample_files(example="famous_quotes", small_only=False) + diff --git a/solutions/gguf/using_function_calls.py b/solutions/gguf/using_function_calls.py new file mode 100644 index 0000000..1b8212d --- /dev/null +++ b/solutions/gguf/using_function_calls.py @@ -0,0 +1,200 @@ + +""" Function Calling with SLIMs - this example illustrates how to move beyond basic question-answer prompting, +and begin to integrate function calls into LLM-based workflows. + + Generally, function-calling is a specialized capability of frontier language models, such as OpenAI GPT4. + + We have adapted this concept to small language models through SLIMs (Structured Language Instruction Models), + which are 'single function' models fine-tuned to accept three main inputs to construct a prompt: + + As of June 2024, there are 18 distinct SLIM function calling models with many more on the way, for most common + extraction, classification, and summarization tasks. + + All SLIM models have a common prompting structure + + Inputs: + -- text passage - this is the core passage or piece of text that you would like the model to assess + -- function - classify, extract, generate - this is handled by default by the model class, so usually does + not need to be explicitly declared - but is an option for SLIMs that support more than one function + -- params - depends upon the model, used to configure/guide the behavior of the function call - optional for + some SLIMs + + Outputs: + -- structured python output, generally either a dictionary or list + + Main objectives: + -- enable function calling with small, locally-running models, + -- simplify prompts by defining specific functions and fine-tuning the model to respond accordingly + without 'prompt magic' + -- standardized outputs that can be handled programmatically as part of a multi-step workflow. + """ + + +from llmware.models import ModelCatalog + + +def discover_slim_models(): + + """ Discover a list of SLIM tools in the Model Catalog. + + -- SLIMs are available in both traditional Pytorch and quantized GGUF packages. + -- Generally, we train/fine-tune in Pytorch and then package in 4-bit quantized GGUF for inference. + -- By default, we designate the GGUF versions with 'tool' or 'gguf' in their names. + -- GGUF versions are generally faster to load, faster for inference and use less memory in most environments.""" + + tools = ModelCatalog().list_llm_tools() + tool_map = ModelCatalog().get_llm_fx_mapping() + + print("\nList of SLIM model tools (GGUF) in the ModelCatalog\n") + + for i, tool in enumerate(tools): + model_card = ModelCatalog().lookup_model_card(tool_map[tool]) + print(f"{i} - tool: {tool} - " + f"model_name: {model_card['model_name']} - " + f"model_family: {model_card['model_family']}") + + return 0 + + +def hello_world_slim(): + + """ SLIM models can be identified in the ModelCatalog like any llmware model. Instead of using + inference method, SLIM models are used with the function_call method that prepares a special prompt + instruction, and takes optional parameters. + + This example shows a series of function calls with different SLIM models. + + Please note that the first time the models will be pulled from the llmware Huggingface repository, and will + take a couple of minutes. Future calls will be much faster once cached in memory locally. """ + + print("\nExecuting Function Call Inferences with SLIMs\n") + + # Sentiment Analysis + + passage1 = ("This is one of the best quarters we can remember for the industrial sector " + "with significant growth across the board in new order volume, as well as price " + "increases in excess of inflation. We continue to see very strong demand, especially " + "in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would " + "be accumulating more stock on any dips.") + + # here are the two key lines of code + model = ModelCatalog().load_model("slim-sentiment-tool") + response = model.function_call(passage1) + + print("sentiment response: ", response['llm_response']) + + # Named Entity Recognition + + passage2 = "Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s." + + model = ModelCatalog().load_model("slim-ner-tool") + response = model.function_call(passage2) + + print("ner response: ", response['llm_response']) + + # Extract anything with Slim-extract + + passage3 = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + + model = ModelCatalog().load_model("slim-extract-tool") + response = model.function_call(passage3, function="extract", params=["revenue growth"]) + + print("extract response: ", response['llm_response']) + + # Generate questions with Slim-Q-Gen + + model = ModelCatalog().load_model("slim-q-gen-tiny-tool", temperature=0.2, sample=True) + # supported params - "question", "multiple choice", "boolean" + response = model.function_call(passage3, params=['multiple choice']) + + print("question generation response: ", response['llm_response']) + + # Generate topic + + model = ModelCatalog().load_model("slim-topics-tool") + response = model.function_call(passage3) + + print("topics response: ", response['llm_response']) + + # Generate headline summary with slim-xsum + model = ModelCatalog().load_model("slim-xsum-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("xsum response: ", response['llm_response']) + + # Generate boolean with optional '(explain)` in parameter + model = ModelCatalog().load_model("slim-boolean-tool") + response = model.function_call(passage3, params=["Did Adobe revenue increase? (explain)"]) + + print("boolean response: ", response['llm_response']) + + # Generate tags + model = ModelCatalog().load_model("slim-tags-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("tags response: ", response['llm_response']) + + return 0 + + +def using_logits_and_integrating_into_process(): + + """ This example shows two key elements of function calling SLIM models - + + 1. Using Logit Information to indicate confidence levels, especially for classifications. + 2. Using the structured dictionary generated for programmatic handling in a larger process. + + """ + + print("\nExample: using logits and integrating into process\n") + + text_passage = ("On balance, this was an average result, with earnings in line with expectations and " + "no big surprises to either the positive or the negative.") + + # two key lines (load_model + execute function_call) + additional logit_analysis step + sentiment_model = ModelCatalog().load_model("slim-sentiment-tool", get_logits=True) + response = sentiment_model.function_call(text_passage) + analysis = ModelCatalog().logit_analysis(response,sentiment_model.model_card, sentiment_model.hf_tokenizer_name) + + print("sentiment response: ", response['llm_response']) + + print("\nAnalyzing response") + for keys, values in analysis.items(): + print(f"{keys} - {values}") + + # two key attributes of the sentiment output dictionary + sentiment_value = response["llm_response"]["sentiment"] + confidence_level = analysis["confidence_score"] + + # use the sentiment classification as a 'if...then' decision point in a process + if "positive" in sentiment_value: + print("sentiment is positive .... will take 'positive' analysis path ...", sentiment_value) + else: + print("sentiment is negative .... will take 'negative' analysis path ...", sentiment_value) + + if "positive" in sentiment_value and confidence_level > 0.8: + print("sentiment is positive with high confidence ... ", sentiment_value, confidence_level) + + return 0 + + +if __name__ == "__main__": + + # discovering slim models in the llmware catalog + discover_slim_models() + + # running function call inferences + hello_world_slim() + + # doing interesting stuff with the output + using_logits_and_integrating_into_process() + + diff --git a/solutions/gguf/using_gguf.py b/solutions/gguf/using_gguf.py new file mode 100644 index 0000000..b5b8617 --- /dev/null +++ b/solutions/gguf/using_gguf.py @@ -0,0 +1,48 @@ + +from llmware.prompts import Prompt +from llmware.models import ModelCatalog + +# Registered by default +# --dragon models: dragon-mistral-7b-gguf | dragon-yi-6b-gguf | dragon-llama-7b-gguf +# --The Bloke leading 7b chat models: llama2-chat | openhermes | zephyr | starling + + +# example 1 - how to use a default gguf model in llmware +def use_default_gguf_model(): + + selected_gguf_model = "llmware/dragon-mistral-7b-gguf" + prompter = Prompt().load_model(selected_gguf_model) + + response = prompter.prompt_main("How old am I?", context="I am 36 years old.") + + print("response: ", response) + + return response + + +response = use_default_gguf_model() + + +# example 2 - how to use any GGUF model from The Bloke on HuggingFace +def register_gguf_model(): + + prompter = Prompt() + + your_model_name = "my_gguf_model_1" + hf_repo_name = "TheBloke/model_name" + model_file = "abc.gguf" + + prompter.model_catalog.register_gguf_model(your_model_name,hf_repo_name, model_file, prompt_wrapper="open_chat") + prompter.load_model(your_model_name) + + return 0 + + +# example 3 - how to use build-from-source custom/optimized llama.cpp +def build_your_own_llama_cpp_lib(): + + import os + os.environ["GGUF_CUSTOM_LIB_PATH"] = "/path/to/your/custom/lib" + + return 0 + diff --git a/solutions/gguf/using_gguf_vision_model.py b/solutions/gguf/using_gguf_vision_model.py new file mode 100644 index 0000000..080fc6c --- /dev/null +++ b/solutions/gguf/using_gguf_vision_model.py @@ -0,0 +1,22 @@ + +""" GGUF Vision Model example - get started with vision-to-text using mtmd tool in conjunction with llama cpp """ + +from llmware.models import ModelCatalog + +model_name = "qwen2.5-vl-3b-instruct-gguf" + +# add path to local image +image_file_path = "/local/path/to/jpg_or_png_image" + +# add text prompt/instruction +prompt = "Describe this image." + +model = ModelCatalog().load_model(model_name, max_output=500) + +# to run streaming generation +for token in model.stream(prompt,image_file_path): + print(token, end="") + +# to run inference (response once completed at the end) +response = model.inference(prompt,image_file_path) +print("--test: inference response: ", response) diff --git a/solutions/gguf/using_local_foundry_models.py b/solutions/gguf/using_local_foundry_models.py new file mode 100644 index 0000000..44b19a3 --- /dev/null +++ b/solutions/gguf/using_local_foundry_models.py @@ -0,0 +1,38 @@ + +""" This example shows how to use Windows Local Foundry models. + + pre-reqs: + + 1. install local foundry, e.g., `winget install Microsoft.FoundryLocal` + 2. pip3 install foundry-local-sdk + 3. pip3 install openai (openai api used, but not openai model - all runs locally) +""" + +from llmware.models import ModelCatalog, WindowsLocalFoundryHandler + +# activate the connection and poll foundry-local instance for available models +foundry_handler = WindowsLocalFoundryHandler() +cat = foundry_handler.activate_catalog(True) + +# foundry local models now added to the catalog +foundry_models = ModelCatalog().list_models_by_type("WindowsLocalFoundryModel") +for i, mod in enumerate(foundry_models): + print("--foundry model - ", mod) + +all_models = ModelCatalog().list_all_models() + +# load foundry model like any other in llmware +# note: this example was used on a Windows Intel x86 Lunar Lake +# -- different platforms will have different supported models + +m1 = "Phi-3.5-mini-instruct-openvino-gpu:1-foundry" +m2 = "qwen2.5-0.5b-instruct-generic-cpu:4-foundry" +model = ModelCatalog().load_model(m1, max_output=500) + +for token in model.stream("What are the best sites to see in Rome?"): + print(token, end="") + +# stop foundry local server when done +WindowsLocalFoundryHandler().stop_server() + + diff --git a/solutions/gguf/using_slim_extract.py b/solutions/gguf/using_slim_extract.py new file mode 100644 index 0000000..9287845 --- /dev/null +++ b/solutions/gguf/using_slim_extract.py @@ -0,0 +1,417 @@ + +""" This example illustrates how to use the slim-extract model to extract custom keys from selected text. + We have included a set of sample earnings releases (comprising lines ~10 - ~385 of this script), and run a + simple loop through the earnings releases, showing how to create an extract prompt to identify + 'revenue growth' from these examples. + + There are several function-calling models in the slim-extract family, fine-tuned on multiple leading + small model base foundations - full list and options are below in the code. """ + +from llmware.models import ModelCatalog + +# Sample earnings releases + +earnings_releases = [ + + {"context": "Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee."}, + + {"context": "Dick’s Sporting Goods raised its dividend by 10% on Thursday as the company posted its largest sales " + "quarter in its history and projected another year of growth. The company’s shares jumped more than " + "15% in intraday trading. CEO Lauren Hobart said on an earnings call Thursday that Dick’s sales " + "growth came from bigger tickets — either higher prices or more expensive items — as its transactions " + "were flat. Many retailers benefited from a 53rd week in fiscal 2023, but Dick’s said it still broke " + "records during its fiscal fourth quarter even without those extra days. Here’s how the athletic " + "apparel retailer did compared with what Wall Street was anticipating, based on a survey of " + "analysts by LSEG, formerly known as Refinitiv: Earnings per share: $3.85 adjusted vs. $3.35 expected " + "Revenue: $3.88 billion vs. $3.80 billion expected The company’s reported net income for the three-month " + "period that ended Feb. 3 was $296 million, or $3.57 per share, compared with $236 million, or $2.60 a " + "share, a year earlier. Excluding one-time items related to impairment charges and inventory write-offs, " + "Dick’s reported earnings per share of $3.85. Sales rose to $3.88 billion, up about 8% from $3.60 billion " + "a year earlier. “With our industry-leading assortment and strong execution, we capped off the year " + "with an incredibly strong fourth quarter and holiday season,” Hobart said in a statement. “We are " + "guiding to another strong year in 2024. We plan to grow both our sales and earnings through " + "positive comps, higher merchandise margin and productivity gains,” she added. During the quarter, " + "same-store sales rose 2.8%, well ahead of the 0.8% lift that analysts had expected, according to " + "StreetAccount. “Growth in transactions” and market share gains drove the increase, said Executive " + "Chairman Ed Stack."}, + + {"context": "Comcast topped both revenue and profit estimates in the fourth quarter as it lost fewer broadband " + "subscribers than expected, and it raised its dividend 7%, the company said Thursday. " + "Here’s how Comcast performed, compared with estimates from analysts surveyed by LSEG, " + "formerly known as Refinitiv. Earnings per share: 84 cents adjusted vs. 79 cents expected " + "Revenue: $31.25 billion vs. $30.51 billion expected For the quarter ended Dec. 31, net " + "income rose 7.8% to $3.26 billion, or 81 cents a share, compared with $3.02 billion, or " + "70 cents a share, a year earlier. Revenue increased 2.3% compared with the prior-year period. " + "Adjusted earnings before interest, taxes, depreciation and amortization (EBITDA) was flat year " + "over year at about $8 billion. 'For the third consecutive year, we generated the highest revenue, " + "adjusted EBITDA and adjusted EPS in our company’s history', Comcast Chief Executive Officer Brian " + "Roberts said in a statement. 'We also reported the highest adjusted EBITDA on record at Theme Parks; " + "were the #1 studio in worldwide box office for the first time since 2015; and maintained Peacock’s " + "position as the fastest growing streamer in the U.S.'"}, + + {"context": "Dollar General forecast annual sales above Wall Street estimates on Thursday, banking on higher " + "demand from inflation-hit customers buying groceries and essentials from the discount retailer’s stores. " + "Shares of the company rose about 6% in early trading, after falling nearly 45% in 2023 on rising costs " + "and stiff competition from bigger retailers. But higher prices and borrowing costs have prompted " + "budget-conscious consumers to cook more meals at home, helping Dollar General record stronger " + "footfall at its outlets as shoppers hunt for lower-margin, needs-based goods, over pricier general " + "merchandise. “With customer traffic growth and market share gains during the quarter, we believe our " + "actions are resonating with customers,” CEO Todd Vasos said in a statement. Vasos’s strategy - to focus " + "on the basics, like more employee presence at stores, greater customer engagement and expanding " + "private-label brands - has helped stabilize Dollar General’s business. Over the last few quarters, " + "Dollar General and rival Dollar Tree have struggled with rising costs linked to their supply " + "chains, labor and raw materials, while facing tough competition from retailers like Walmart " + "and Chinese ecommerce platform Temu. Dollar Tree’s shares fell more than 15% on Wednesday, after it " + "forecast weak sales and profit for 2024 and laid out plans to shutter 970 of its Family Dollar " + "stores. “Dollar General has a much rosier outlook than Dollar Tree... Dollar Tree’s challenges " + "with Family Dollar were years in the making, while Dollar General has embarked on an aggressive " + "effort to add more frozen, refrigerated and fresh produce,” eMarketer senior analyst Zak Stambor said. " + "Dollar General forecast 2024 sales to grow between 6.0% and 6.7%, above analysts’ estimate of 4.4% " + "growth to $40.33 billion, according to LSEG data. It still sees annual per-share profit between " + "$6.80 and $7.55, compared with estimates of $7.55. Its fourth-quarter net sales of $9.86 billion " + "surpassed estimates of $9.78 billion. It also reported an estimate-beating profit of $1.83 per share."}, + + {"context": "Shares of Zara owner Inditex hit record highs on Wednesday according to LSEG data, climbing over 6% during " + "intraday trading after the company announced its 2023 full-year results. As of 11:50 London time, shared " + "were just over 6% higher at 43.58 euros, or $47.69. Sales increased 10.4% to 35.9 billion euros for the " + "year, the company said, signaling this was a record high. Sales grew across all geographic regions and " + "across Inditex’s brands and were “very satisfactory,” both online and in store, the company said. A total of " + "5,692 stores were operational at the end of the year, Inditex said, adding it plans to expand further in " + "2024, including with Zara shops in Los Angeles and Las Vegas. The company also plans to open new distribution " + "centers in 2024 and 2025, as part of a major logistics expansion plan that will cost the company " + "investments of 900 million euros in both years. Net income also reached a fresh high after soaring " + "30.3% from 2022 to reach 5.4 billion euros last year. The company’s gross profit came in at 20.8 billion " + "euros, up 11.9% on the year. “Inditex’s performance in 2023 has been excellent. Our teams have been able to " + "take advantage of the opportunities to keep growing profitably. We are investing to drive future growth and " + "continue to offer an attractive remuneration to shareholders,” Inditex CEO Oscar García Maceiras said in a " + "statement. The Spanish clothing company owns a range of vastly popular brands including household name " + "Zara, as well as Pull & Bear, Bershka, Stradivarius, premium retailer Massimo Dutti and sports and the " + "athleisure-focused Oysho. Zara, including the Zara Home range, was the biggest contributor to sales in " + "2023, followed by Pull & Bear and Massimo Dutti, Inditex said Wednesday. The company also indicated " + "that 2024 was off to a strong start, with sales in constant currency up 11% over the Feb. 1 to March 11 " + "stretch, compared with the same period a year earlier."}, + + {"context": "Oracle reported quarterly earnings on Monday that exceeded Wall Street’s expectations. Shares rose " + "13% in extended trading. Here’s how the company did in the fiscal third quarter ending Feb. 29, compared " + "to estimates by LSEG, formerly known as Refinitiv: Earnings per share: $1.41 adjusted vs. $1.38 expected " + "Revenue: $13.28 billion vs. $13.3 billion expected For the fiscal fourth quarter, Oracle said it expects " + "earnings of $1.62 to $1.66 per share. Analysts were expecting $1.64 in adjusted earnings per share, according " + "to LSEG. Revenue growth will be between 4% and 6% over sales of $13.8 billion a year ago. The midpoint of that " + "range would equal revenue of about $14.5 billion, while analysts were expecting a little more than $14.7 billion. " + "Oracle CEO Safra Catz said the company was committed to hitting previously stated goals of $65 billion in " + "sales by fiscal 2026. “Some of these goals might prove to be too conservative given our momentum,” Catz said. " + "Revenue rose 7% in the quarter from $12.4 billion a year earlier. Net income climbed 27% to $2.4 billion, " + "or 85 cents per share, from $1.9 billion, or 68 cents per share, a year ago. Oracle’s cloud services and " + "license support segment, its largest business, saw sales rise 12% to $9.96 billion, slightly beating " + "StreetAccount consensus expectations of $9.94 billion. The company attributed the rise to strong demand " + "for its artificial intelligence servers. Catz said the company added several “large new cloud " + "infrastructure” contracts during the quarter. The company’s cloud revenue, which is reported as part " + "of the cloud services unit, rose 25% year over year to $5.1 billion, Oracle said. “We signed several large " + "deals this quarter and we have many more in the pipeline,” Catz told investors on the earnings call. " + "Oracle Chairman Larry Ellison cited increased business from Microsoft on the earnings call. " + "“We’re building 20 data centers from Microsoft and Azure. They just ordered three more data centers " + "this week,” Ellison said. The company’s other units didn’t fare as well. Cloud license and on-premise sales " + "declined 3% to $1.26 billion, slightly beating StreetAccount’s forecast. Hardware revenue fell 7% to " + "$754 million, while sales in the company’s services division slid 5% to $1.31 billion, both falling short " + "of StreetAccount expectations. Prior to Monday’s report, Oracle shares were up 8.7% for the year, " + "slightly outperforming the S&P 500."}, + + {"context": "Porsche on Tuesday warned that profitability will decline this year as it launches new models amid " + "tough economic conditions, but hiked its dividend on the back of a rise in 2023 operating profit. The German " + "luxury automaker said it expects an operating return on sales of between 15% and 17% in 2024, down from the " + "18% margin notched in 2023 and 2022. In the long term, the group targets an operating return on sales of more " + "than 20%. Explaining the more cautious profitability outlook, the company cited “the comprehensive renewal " + "of its product range in 2024, the global framework conditions, higher depreciations on capitalized " + "development costs and the continued investments in the brand and the Porsche ecosystem.” The company’s shares " + "were around 4.8% higher by early afternoon, having reversed opening losses of more than 2%. Porsche is " + "launching four new car ranges in 2024 in the form of the Panamera, Macan, Taycan and 911 model lines. " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China WATCH NOW VIDEO 03:17 " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China “2024 is going to be a year of " + "product launches for Porsche – more so than any year in our history,” Chairman Oliver Blume said in a " + "statement. “We will be introducing a variety of exhilarating sports cars to the road, they will delight " + "our customers around the world. This will put the wind at our back for years to come.”"}, + + {"context": "Lego on Tuesday reported its full-year 2023 results, saying it’s revenue grew by 2% throughout the year, " + "in line with expectations. The company made “very, very strong progress” and “grew comfortably” in the " + "U.S., its CEO Niels Christiansen told CNBC. The toy industry has been struggling to maintain " + "pandemic-era growth as inflation is putting pressure on demand and sales. In-store participation is greater " + "than prior to the pandemic, Lego CEO says In-store participation is greater than prior " + "to the pandemic, Lego CEO says The chief executive of Denmark’s Lego on Tuesday reflected on a tough year " + "for the world’s largest toymaker, and outlined the firm’s long-term plans to stay relevant and “cool with kids. ”" + "Lego said its 2023 revenue was 2% higher compared to the previous year, growing to 65.9 billion Danish krone " + "(around $9.65 billion). This was in line with expectations, Lego said in a statement. “It was a difficult year,” " + "Lego CEO Niels Christiansen told CNBC. However, he said the company had “managed to take quite a bit of " + "market share.” The Danish toymaker said operating profit declined slightly from 17.9 billion Danish krone " + "to 17.1 billion, noting that it had boosted spending on strategic initiatives designed to drive growth. " + "Net profit came in at 13.1 billion Danish krone in 2023, compared to 13.8 billion the previous year. " + "Consumer sales were up 4% despite slumping in China, Lego said, attributing the growth to increasing demand " + "in the U.S. and central and eastern Europe. It comes as the wider toy industry has been struggling to " + "maintain growth after booming during the coronavirus pandemic, when parents looked for new ways to " + "entertain their children and adults re-discovered childhood pastimes. Toy company Hasbro earlier this month " + "said its 2023 revenue fell by 15% compared to 2022 and that it expected to see a further decline this year."}, + + {"context": "Adidas on Wednesday warned of a sales decline in its overstocked North American market in 2024, as the " + "German sportswear brand continues to sell off its remaining Yeezy inventory. Currency-neutral sales in " + "North America are expected to decline to a mid-single-digit rate in 2024, but are projected to notch " + "mid-single-digit growth worldwide despite persistent “macroeconomic challenges and geopolitical tensions,” " + "the company said. Adidas confirmed its 2023 operating profit came in at 268 million euros ($292.9 million) " + "on the back of flat currency-neutral sales, significantly above prior expectations as the company continues " + "to take a hit from the cessation of its line of Yeezy — footwear the retailer produced in a collaboration with " + "American rapper Ye, formerly known as Kanye West. For the fourth quarter, the company posted an operating " + "loss of 377 million euros. The board proposed a flat dividend of 0.70 euros per share. “Although by far not " + "good enough, 2023 ended better than what I had expected at the beginning of the year,” CEO Bjørn Gulden " + "said in a statement. “Despite losing a lot of Yeezy revenue and a very conservative sell-in strategy, " + "we managed to have flat revenues. We expected to have a substantial negative operating result, but " + "achieved an operating profit of €268 million.” Adidas was confirming preliminary results released in late " + "January, when it announced that it would not write off the majority of its Yeezy inventory and would instead " + "sell off the remaining shoes at cost. The sportswear giant was forced to axe the Yeezy line after terminating " + "its partnership with Ye over a string of anti-Semitic remarks that the rapper made in 2022. Adidas said the " + "discontinuation of Yeezy represented a drag of around 500 million euros in the year-on-year comparison " + "through 2023, though the sale of parts of the remaining inventory in the second and third quarter positively " + "impacted net sales by around 750 million euros. “With a very disciplined go-to-market and buying process, " + "we reduced our inventories by almost €1.5 billion. With the exception of the U.S., we now have healthy " + "inventories everywhere,” Gulden said. He added that the company is expecting some growth in the " + "first quarter of 2024 and a further pick-up in the second half of the year. “We still have a lot of work " + "to do, but I feel very confident we are on the right track. We will bring adidas back again. Give us some " + "time and we will again say – we got this!” he said. Adidas projected an operating profit of around " + "500 million euros in 2024, with unfavorable currency effects expected to “weigh significantly on the " + "company’s profitability” because of adverse impacts on both reported revenues and gross margin development." + "Adidas shares were flat by mid-morning on Wednesday. Mamta Valechha, equity research analyst at " + "Quilter Cheviot, said that, given that the headline numbers were already pre-released in January, the most " + "interesting aspect of Wednesday’s report was the “clear acceleration of the Adidas brand.”"}, + + {"context": "Costco on Thursday missed Wall Street’s revenue expectations for its holiday quarter, despite reporting " + "year-over-year sales growth and strong e-commerce gains. Shares of the retailer fell about 4% in aftermarket " + "trading. The company’s stock had hit a 52-week high earlier in the day. Here’s what Costco reported for its " + "fiscal second quarter of 2024 compared with what Wall Street was expecting, based on a survey of analysts by " + "LSEG, formerly known as Refinitiv: Earnings per share: $3.92 vs. $3.62 expected Revenue: $58.44 billion vs. " + "$59.16 billion expected In the three-month period that ended Feb. 18, Costco’s net income rose to " + "$1.74 billion, or $3.92 per share, compared with $1.47 billion, or $3.30 per share, a year earlier. " + "Costco’s revenue for the quarter increased from $55.27 billion in the year-ago period. Comparable sales for " + "the company increased 5.6% year over year and 4.3% in the U.S. Excluding changes in gas prices and foreign " + "currency, the metric increased 5.8% overall and 4.8% in the U.S. Sales of food and sundries, a category " + "that includes snack foods and beverages, were up by mid single digits in the quarter, CFO Richard Galanti " + "said on the company’s earnings call. Fresh foods were up high single digits and nonfoods were up mid single " + "digits. Ancillary businesses, which includes more service-related purchases like travel, were up by low " + "single digits, he said. Costco’s food court, pharmacy and optical centers were top performers in the quarter " + "and gas was down low single digits as the price per gallon fell. More shoppers came to Costco, and they " + "spent more on their shopping trips during the quarter. Traffic increased 5.3% across the globe and 4.3% in " + "the U.S., Galanti said on the earnings call. The average ticket increased in the U.S. and worldwide, he " + "said. Inflation was roughly flat year over year in the quarter, which allowed the retailer to reduce " + "prices for some items, Galanti said. For example, he said, it’s been able to cut the price of reading " + "glasses from $18.99 to $16.99 and slash the price of a 48 count of Kirkland Signature batteries from " + "$17.99 to $15.99. In the prior quarter, he said inflation was as much as 1% year over year. Galanti said many " + "new items in categories like sporting goods and lawn and garden will also have lower prices compared with " + "a year ago because of falling freight and commodity costs. Costco has 875 warehouses, including 603 in " + "the U.S. and Puerto Rico. It also has clubs in about a dozen other countries, including Canada, Mexico, " + "Japan and China. In the second quarter, Costco opened four new clubs, including three in the U.S. " + "and one in Shenzhen, China. That marked its sixth club to open in China, Galanti said. Two of the three " + "new U.S. locations were Costco Business Centers, which are specifically geared toward small business " + "owners like restaurant operators. As of Thursday’s close, Costco shares have risen nearly 19% since the " + "start of the year. The stock touched a 52-week high of $787.08 earlier in the day and closed at $785.59, " + "bringing the company’s market value to nearly $350 billion."}, + + {"context": "Shares of Teleperformance plunged 23% on Thursday, after the French call center and office services group " + "missed its full-year revenue target and flagged a “volatile economic environment.” Investors have been " + "spooked by the potential impact of artificial intelligence on its business model, as companies become more " + "able to tap into the technology directly for their own benefit. Teleperformance shares dropped 16% last " + "week, according to LSEG data, after Swedish financial services company Klarna said its Open AI-powered " + "customer service assistant was handling two-thirds of customer service calls. But Teleperformance CEO " + "Daniel Julien on Thursday said that AI would be a positive for its business model — and that it will never " + "fully replace the value of human interaction. “AI is part of the solutions we provide to the clients,” " + "Julien told CNBC’s “Squawk Box Europe.” “AI helps to increase the accuracy of our employees ... which is " + "great, but at the end of the day we are here to reduce the friction between the citizens, or the customer, " + "and the companies they have bought a product and service from.” He stressed, “It’s not just a transactional " + "relationship, it has a lot to do with reassuring, with trust, with empathy. So we perceive AI as enhancing " + "the job that our human employees do, but absolutely not replacing them.” hide content Teleperformance SE " + "RT Quote | Exchange | EUR 87.16 quote price arrow up+0.68 (+0.79%) Last | 03/15/24 CET WATCHLIST + QUOTE DETAILS " + "Teleperformance share price. Teleperformance reported 2.3% higher revenue at 8.345 billion euros " + "($9.091 billion) in 2023, as net profit fell year-on-year from 643 million euros to 602 million euros. " + "Diluted earnings per share hit 10.18 euros, down from 10.77 euros. In its results, the company said it is " + "working with clients on 250 AI projects, including in generative AI, and it has expanded its portfolio " + "with new partnerships in the space. “Even the most high-tech or the most AI-involved companies are clients " + "of Teleperformance. We chose that there is a complementarity and not separation,” Julien told CNBC, " + "flagging the company’s agreement with tech giant and major AI player Microsoft. “They are there to provide " + "a solution that is going to augment the productivity, augment the quality of the information that can be " + "given to the customer, but, at the end of the day, the customer is a human being. The day the customer is " + "going to be a robot, maybe AI will replace the humans.”"}, + + {"context": "CrowdStrike shares surged as much as 21% in after-hours trading Tuesday after the cybersecurity " + "company reported a beat on the top and bottom lines, plus issued stronger-than-expected guidance for " + "the upcoming quarter and full year. Here’s how the company did compared to consensus estimates " + "based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per share: 95 cents " + "adjusted vs. 82 cents expected Revenue: $845 million vs. $839 million expected For the period that " + "ended Jan. 31, CrowdStrike saw net income of $54 million, or 22 cents per share, from a " + "$48 million loss, or a 20 cent loss per share, in the year-ago period. CrowdStrike has now " + "reported GAAP net income for the past four quarters, Chief Financial Officer Burt Podbere " + "said in the earnings release. Full-year revenue rose 36% year over year, from $2.24 billion " + "to $3 billion. The company also announced it would acquire Flow Security for an undisclosed " + "price in a cash-and-stock deal, slated to close in the company’s fiscal first quarter. " + "The company has been stepping up its merger and acquisition activity in recent months. “CrowdStrike " + "is cybersecurity’s consolidator of choice, innovator of choice, and platform of choice to " + "stop breaches,” co-founder and CEO George Kurtz said in a release. The company also guided to " + "fiscal first-quarter revenue between $902 million and $906 million, better than a consensus " + "estimate of $899 million. CrowdStrike also expects earnings per share for the period between " + "89 cents and 90 cents, better than the consensus estimate of 82 cents. Podbere also reiterated " + "the company’s focus on achieving $10 billion in annual recurring revenue by 2030. The company " + "reached $3.4 billion in annual recurring revenue in January."}, + + {"context": "Shares of Amer Sports, the maker of Wilson tennis rackets and Lousiville Slugger baseball " + "bats, fell on Tuesday after the company reported strong sales in China but a slowdown in " + "wholesale orders. Here’s how the newly public athletic company did in its fourth quarter. " + "CNBC didn’t compare the results to Wall Street estimates because it’s the first earnings " + "report since Amer Sports went public. Loss per share: 25 cents Revenue: $1.32 billion In the " + "three months ended Dec. 31, the company reported a net loss of $94.9 million, or 25 cents " + "per share, compared with $148.3 million, or 39 cents per share, a year earlier. Sales rose to " + "$1.32 billion, up about 10% from $1.2 billion a year earlier. Shares closed about 5% lower. " + "Amer, which also owns Arc’teryx, Salomon, and a number of other athletic equipment and " + "apparel brands, operates in three distinct business segments. They are technical apparel, " + "which includes its pricey Arc’teryx winter jackets; outdoor performance, such as Salomon’s " + "winter sports equipment; and ball and racquet sports, which includes equipment and apparel " + "from Wilson and Louisville, among others. During the quarter, sales for Amer’s technical " + "apparel rose 26% year over year to $550 million, driven by a 42% jump in direct sales. " + "Sales in the segment primarily come from shoppers who are buying directly from Amer’s " + "brands rather than from wholesale partners. Sales for outdoor performance increased 2% to " + "$523 million, driven by strength in the segment’s winter sports equipment franchise, " + "which was offset by a slowdown in wholesale orders for Salomon footwear. Ball and racquet sales " + "declined 3% to $242 million as the segment lapped tougher comps. In the year-ago period, " + "retailers were still dealing with supply chain issues and had over-ordered equipment like " + "tennis rackets and baseball bats. As they looked to keep their inventory levels in check, " + "some wholesalers pulled back on orders during the quarter, but Amer expects the segment " + "will level out in the quarters ahead and end fiscal 2024 with sales up in the low- to " + "mid-single digit range. The company started trading on the New York Stock Exchange last " + "month under the ticker “AS.” The shares rose just 3% in Amer’s debut on the public " + "markets after it priced its IPO at a discount. Sellers showed muted interest in the " + "stock during its first day of trading over concerns about its connections and exposure to " + "China and its debt-laden balance sheet. Founded in Helsinki in 1950, Amer was a Finnish public " + "company until it was taken private in 2019 by a consortium of investors led by China’s Anta " + "Sports, FountainVest Partners, Anamered Investments and Tencent. Since the acquisition, " + "sales grew about 45% from $2.45 billion in 2020 to $3.55 billion in 2022. Revenue jumped " + "again in 2023 to $4.37 billion, the company said Tuesday."}, + + {"context": "Shares of Dell Technologies popped more than 15% during extended trading Thursday after the company " + "released fourth-quarter results that beat analysts’ estimates and showed strong demand for its " + "artificial intelligence servers. Here’s how the company did: Earnings per share: $2.20 adjusted vs. " + "$1.73 expected by LSEG, formerly known as Refinitiv Revenue: $22.32 billion vs. $22.16 billion " + "expected by LSEG Dell’s revenue for the fiscal 2024 fourth quarter fell 11% from $25.04 billion " + "in the year-ago quarter. The company reported net income $1.16 billion, up 89% from the $614 million " + "it posted in the same period last year. Chief Financial Officer Yvonne McGill said in a release " + "that the company is increasing its annual dividend by 20% to $1.78 per share, which she called " + "a “testament to our confidence in the business.” Dell’s Infrastructure Solutions Group (ISG) " + "reported $9.3 billion in revenue for the quarter, down 6% year over year but up 10% from the " + "third quarter. Servers and networking revenue made up the bulk of that, with $4.9 billion in " + "revenue driven by “AI-optimized servers.” Storage revenue came in at $4.5 billion. The company’s " + "Client Solutions Group (CSG) reported $11.7 billion for the quarter, down 12% year over year. " + "That includes $9.6 billion in commercial client revenue, which fell 11% since the fourth quarter " + "of last year, and $2.2 billion in consumer revenue, down 19% year over year. “Our strong AI-optimized " + "server momentum continues, with orders increasing nearly 40% sequentially and backlog nearly " + "doubling, exiting our fiscal year at $2.9 billion,” Chief Operating Officer Jeff Clarke " + "said in the release. For its first quarter, Dell said during its quarterly call with " + "investors that it expects to report revenue between $21 billion and $22 billion. The company " + "said it is encouraged by momentum around AI, and that it expects to return to growth for " + "fiscal 2025. However, the company noted that the macroeconomic environment is causing some " + "customers to be cautious about infrastructure costs."}, + + {"context": "Birkenstock on Thursday beat holiday quarter revenue expectations, reporting a 22% year-on-year " + "jump, as the German sandal company benefited from higher pricing and rising U.S. demand. As a newly " + "public company, Birkenstock is still getting into a public reporting rhythm and only just " + "released its fiscal 2023 results and 2024 guidance a little over a month ago. On Thursday, " + "it said it stands by guidance issued then and still expects sales to be between 1.74 billion " + "euros and 1.76 billion euros ($1.89 billion and $1.91 billion), representing growth of 17% to 18%. " + "The shoemaker, which started trading on the New York Stock Exchange under the ticker “BIRK” in " + "October, saw a muted debut when it first hit the public markets, with shares sliding more than " + "12% on its first day as a public company. The stock has since rebounded and is up more than 5% " + "this year, as of the Wednesday close. Birkenstock’s shares closed more than 2% lower Thursday. " + "Here’s how the shoemaker did in its fiscal first quarter compared with what Wall Street was " + "anticipating, based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per " + "share: 9 euro cents adjusted vs. 9 euro cents expected. Revenue: 302.9 million euros vs. 288.7 " + "million euros expected. The company reported a net loss of 7.15 million euros for the " + "three-month period that ended Dec. 31, or a loss of 4 euro cents per share. A year earlier, " + "it reported a loss of 9.19 million euros, or a loss of 5 euro cents per share. " + "Excluding one-time items, Birkenstock reported a profit of 17 million euros, or " + "9 euro cents per share. Sales rose to 302.9 million euros, up 22% from 248.5 million euros " + "a year earlier. Adjusted earnings before interest, taxation, depreciation and amortization " + "(EBITDA) rose 12% year on year to 81 million euros, with an adjusted EBITDA margin of 26.9%, " + "down from 29.1% a year earlier. The retailer has been making strides to grow its direct-to-consumer " + "business, which comes with better profits and more customer insights than relying on wholesale partners. " + "CEO Oliver Reichert has said the company deliberately engineers its distribution strategy so " + "demand is higher than supply but it’s working to double its production capabilities over " + "the next three years to narrow that gap. The chief executive said those investments, " + "along with other efforts the company is undertaking to drive growth, is having a “planned” " + "but “temporary” impact to profitability. The company’s gross profit margin inched down to " + "61% from 61.7% during the same period last year, with Birkenstock citing “unfavorable " + "currency translation and the planned, temporary under-absorption from our ongoing " + "capacity expansion.” The company said it continues to carefully track input costs and " + "is mitigating inflationary pressures with “executed, selective price increases.” In Europe, the " + "company said it had “two consecutive price adjustments” with “no signs of rejection.”"}, + + {"context": "Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter."} + +] + +# *** Execution script starts here *** + +# specialized function-calling extraction models on top of several leading small model bases, +# ranging from 0.5b (qwen2) - 3.8b (phi3) + +slim_extract_models = ["slim-extract-tool", # original - stablelm-3b (2.7b) + "slim-extract-tiny-tool", # tiny-llama 1.1b + "slim-extract-qwen-1.5b-gguf", # **NEW** qwen 1.5b + "slim-extract-phi-3-gguf", # **NEW** phi-3 (3.8b) + "slim-extract-qwen-0.5b-gguf"] # **NEW** qwen 0.5b + +# load the model +model = ModelCatalog().load_model("slim-extract-tool",sample=False,temperature=0.0, max_output=100) + +# iterate through the earnings release samples above +for i, sample in enumerate(earnings_releases): + + # key line: execute function_call on selected model with 'custom_key' = "revenue growth" + response = model.function_call(sample["context"], function="extract", params=["revenue growth"]) + + # display the response on the screen + print("extract response: ", i, response["llm_response"]) + diff --git a/solutions/models/adjusting_sampling_settings.py b/solutions/models/adjusting_sampling_settings.py new file mode 100644 index 0000000..16b255c --- /dev/null +++ b/solutions/models/adjusting_sampling_settings.py @@ -0,0 +1,52 @@ + +""" This example illustrates how to adjust sampling parameters when loading a model to analyze the impact of + sampling on token selection from the model. + -- note: these parameters are implemented and designed for locally deployed models, e.g., HFGenerativeModel class + and GGUFGenerativeModel class. + -- note: we have seen for function-calling, in particular, that turning sample=False generally yields better + and more consistent results. +""" + +from llmware.models import ModelCatalog + +sample = "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street "\ + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering"\ + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n"\ + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days."\ + "If you have any questions concerning this invoice, contact Bia Hermes. "\ + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000" + + +# the objective of the example is to run several times, and adjust the following parameters to experiment: +# -- sample: True or False +# -- temperature: range between 0.0 - 1.0 (for GGUF models, you can also try setting to negative) +# -- using get_logits and max_output configuration variables + + +# load model and configure sampling parameters +model = ModelCatalog().load_model("bling-stablelm-3b-tool", + sample=False, + temperature=0.0, + get_logits=True, + max_output=123) + +# run a basic summary inference +response = model.inference("What is a list of the key points?", sample) + +# analyze the sampling +sampling_analysis = ModelCatalog().analyze_sampling(response) + +# display the response +print("response: ", response) + +# display the logits +print("logits: ", response["logits"]) + +# show the sampling analysis +print("sampling analysis: ", sampling_analysis) + +# optional (for more detail) - look 'token-by-token' at 'not_top_tokens' selected due to sampling impact +for i, entries in enumerate(sampling_analysis["not_top_tokens"]): + print("sampled choices: ", i, entries) + + diff --git a/solutions/models/bling_fast_start.py b/solutions/models/bling_fast_start.py new file mode 100644 index 0000000..668cdca --- /dev/null +++ b/solutions/models/bling_fast_start.py @@ -0,0 +1,417 @@ + +""" This example demonstrates prompting local BLING models with provided context - easy to select among different +BLING models between 1B - 4B, including both Pytorch versions and GGUF quantized versions, and to swap out the +hello_world questions with your own test set. + + NOTE: if you are running on a CPU with limited memory (e.g., <16 GB of RAM), we would recommend sticking to +the 1B parameter models, or using the quantized GGUF versions. You may get out-of-memory errors and/or very +slow performance with ~3B parameter Pytorch models. Even with 16 GB+ of RAM, the 3B Pytorch models should run but +will be slow (without GPU acceleration). """ + + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +def bling_meets_llmware_hello_world (model_name): + + """ Simple inference loop that loads a model and runs through a series of test questions. """ + + t0 = time.time() + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"] + , prompt_name="default_with_context",temperature=0.30) + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # list of 'rag-instruct' laptop-ready bling models on HuggingFace + + model_list = ["llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3", + + # use GGUF models too + "bling-phi-3-gguf", # quantized bling-phi-3 + "bling-answer-tool", # quantized bling-tiny-llama + "bling-stablelm-3b-tool" # quantized bling-stablelm-3b + ] + + # try the newest bling model - 'tiny-llama' + bling_meets_llmware_hello_world(model_list[1]) + diff --git a/solutions/models/chat_models_gguf_fast_start.py b/solutions/models/chat_models_gguf_fast_start.py new file mode 100644 index 0000000..00f91b6 --- /dev/null +++ b/solutions/models/chat_models_gguf_fast_start.py @@ -0,0 +1,103 @@ +"""This example demonstrates several leading open source chat models running in 4-bit GGUF on local laptop.""" + +import time +import re +from llmware.prompts import Prompt +import logging + + +# Run the benchmark test +def run_test(model_name: str, prompt_list: list[dict]) -> int: + """Run the benchmark test on the specified model with the given prompts. + + Args: + model_name (str): The name of the model to load. + prompt_list (list[dict]): A list of prompts to test the model with. + + Returns: + int: Status code (0 for success). + """ + logging.basicConfig(level=logging.INFO) + + logging.info(f"Loading model '{model_name}'") + + try: + prompter = Prompt().load_model(model_name) + except Exception as e: + logging.error(f"Failed to load model: {e}") + return 1 + + for i, entry in enumerate(prompt_list): + start_time = time.time() + logging.info(f"query - {i+1} - {entry['query']}") + + try: + response = prompter.prompt_main(entry["query"]) + except Exception as e: + logging.error(f"Error during prompting: {e}") + continue + + # Print results + time_taken = round(time.time() - start_time, 2) + llm_response = re.sub("[\n\n]", "\n", response['llm_response']) + logging.info(f"llm_response - {i+1} - {llm_response}") + logging.info(f"time_taken - {i+1} - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + # Example open-ended queries with no context - looking for chat model to draw on general know-how and ability + # to look at a problem conceptually, with focus on language understanding without specific focus on facts + + ds = [ + {"query": "I am interested in gaining an understanding of the banking industry. What topics should I research?", + "context": "", "answer": "NO_GOLD_ANSWER"}, + {"query": "What are some tips for creating a successful business plan?", "context": "", "answer": ""}, + {"query": "What do you think about the recent news about Microsoft\u2019s GitHub acquisition?", "context": "", + "answer": ""}, + {"query": "What are the best books to read for a class on American literature?", "context": "", "answer": ""}, + {"query" : "What is the most important thing I should know about my local school district?", "context": "", "answer": ""}, + {"query": "Can you recommend some good books for me to read?", "context": "", "answer": ""}, + {"query": "What are the differences between the four principal cloud computing service models: IaaS, PaaS, SaaS, and CaaS?", + "context": "", "answer": ""}, + {"query": "I've heard a lot of good things about the iPhone. Should I buy one?", "context": "", "answer": ""}, + {"query": "What is the difference between a sociological and psychological approach to social problems?", + "context": "", "answer": ""}, + {"query": "What job opportunities are available for someone with degree in chemistry?", "context": "", "answer": ""}, + {"query": "I'd like to know how to get the most out of my money in the stock market?", "context": "", "answer": ""}, + {"query": "How do I know if my computer is secure?", "context": "", "answer": ""}, + {"query": "I'm writing an essay on the importance of reading. What are some good questions to ask " + "in this type of essay?", "context": "", "answer": ""}, + {"query": "What is the best way for small businesses to raise capital for their operations?", "context": "", "answer": ""}, + {"query": "I want to start a blog but don't know what to write about. What should I do?", "context": "", "answer": ""}, + {"query": "What are the best ways to learn a new language?", "context": "", "answer": ""}, + {"query": "I'm having some problems with my computer. What are some of the most common computer problems" + " and how can I fix them?", "context": "", "answer": ""}, + {"query": "How can I improve my credit score?", "context": "", "answer": ""}, + {"query": "How can I build confidence in my ability to write articles and be a good writer?", + "context": "", "answer": ""}, + {"query": "How do I negotiate a salary raise?", "context": "", "answer": ""}, + {"query": "What's the best way to learn how to write a good essay?", "context": "", "answer": ""}, + {"query": "I'm a little nervous about taking my first trip abroad. What do I need to know?", "context": "", + "answer": ""}, + {"query": "What is the difference between a dividend and a capital gain?", "context": "", "answer": ""}, + {"query": "What are the best ways to make a house a home?", "context": "", "answer": ""}, + {"query": "I have a question about ecology. What is the difference between a population and a community?", + "context": "", "answer": ""} + ] + + # please note that these models will produce multiple bulletpoints and paragraph length answers, so + # it can take ~30 seconds for each response on a typical Mac M1 laptop + + # for full citations and links, please go to www.huggingface.co/llmware/bonchon model repository + # -- TheBloke/OpenHermes-2.5-Mistral-7B-GGUF + # -- TheBloke/zephyr-7B-beta-GGUF + # -- TheBloke/Starling-LM-7B-alpha-GGUF + # -- TheBloke/Llama-2-7B-Chat-GGUF + # + # -- Llama-3: bartowski/Meta-Llama-3-8B-Instruct-GGUF + # -- Phi-3: microsoft/Phi-3-mini-4k-instruct-gguf + + output = run_test("bartowski/Meta-Llama-3-8B-Instruct-GGUF",ds) diff --git a/solutions/models/cohere_command_r_example.py b/solutions/models/cohere_command_r_example.py new file mode 100644 index 0000000..0f56b2b --- /dev/null +++ b/solutions/models/cohere_command_r_example.py @@ -0,0 +1,24 @@ +""" Example script for using the Cohere Command R model. """ + +from llmware.models import ModelCatalog + +def main(): + """ Demonstrates loading and using the Cohere Command R model. """ + + model_name = "command-r" + + # Load the model using ModelCatalog + model = ModelCatalog().load_model(model_name) + + # Define a prompt for inference + prompt = "Explain the theory of relativity in simple terms." + + # Perform inference + response = model.inference(prompt) + + # Print the response + print("LLM Response:", response["llm_response"]) + print("Usage:", response["usage"]) + +if __name__ == "__main__": + main() diff --git a/solutions/models/document_summarizer.py b/solutions/models/document_summarizer.py new file mode 100644 index 0000000..d71d101 --- /dev/null +++ b/solutions/models/document_summarizer.py @@ -0,0 +1,77 @@ + +""" This Example shows a packaged 'document_summarizer' prompt using the slim-summary-tool. It shows a variety of +techniques to summarize documents generally larger than a LLM context window, and how to assemble multiple source +batches from the document, as well as using a 'query' and 'topic' to focus on specific segments of the document. """ + +import os + +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def test_summarize_document(example="jd salinger"): + + # pull a sample document (or substitute a file_path and file_name of your own) + sample_files_path = Setup().load_sample_files(over_write=False) + + topic = None + query = None + fp = None + fn = None + + if example not in ["jd salinger", "employment terms", "just the comp", "un resolutions"]: + print ("not found example") + return [] + + if example == "jd salinger": + fp = os.path.join(sample_files_path, "SmallLibrary") + fn = "Jd-Salinger-Biography.docx" + topic = "jd salinger" + query = None + + if example == "employment terms": + fp = os.path.join(sample_files_path, "Agreements") + fn = "Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + topic = "executive compensation terms" + query = None + + if example == "just the comp": + fp = os.path.join(sample_files_path, "Agreements") + fn = "Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + topic = "executive compensation terms" + query = "base salary" + + if example == "un resolutions": + fp = os.path.join(sample_files_path, "SmallLibrary") + fn = "N2126108.pdf" + # fn = "N2137825.pdf" + topic = "key points" + query = None + + # optional parameters: 'query' - will select among blocks with the query term + # 'topic' - will pass a topic/issue as the parameter to the model to 'focus' the summary + # 'max_batch_cap' - caps the number of batches sent to the model + # 'text_only' - returns just the summary text aggregated + + kp = Prompt().summarize_document_fc(fp, fn, topic=topic, query=query, text_only=True, max_batch_cap=15) + + print(f"\nDocument summary completed - {len(kp)} Points") + for i, points in enumerate(kp): + print(i, points) + + return 0 + + +if __name__ == "__main__": + + print(f"\nExample: Summarize Documents\n") + + # 4 examples - ["jd salinger", "employment terms", "just the comp", "un resolutions"] + # -- "jd salinger" - summarizes key points about jd salinger from short biography document + # -- "employment terms" - summarizes the executive compensation terms across 15 page document + # -- "just the comp" - queries to find subset of document and then summarizes the key terms + # -- "un resolutions" - summarizes the un resolutions document + + summary_direct = test_summarize_document(example="employment terms") + + diff --git a/solutions/models/dragon_gguf_fast_start.py b/solutions/models/dragon_gguf_fast_start.py new file mode 100644 index 0000000..f00b4d6 --- /dev/null +++ b/solutions/models/dragon_gguf_fast_start.py @@ -0,0 +1,75 @@ + +"""This example demonstrates running a 7B RAG-instruct fine-tuned DRAGON model locally on a laptop. + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + + +import time +from llmware.prompts import Prompt +from llmware.exceptions import LLMWareException +from importlib import util +if not util.find_spec("datasets"): + raise LLMWareException(message="\nto run this example, you need to install HuggingFace datasets: " + "`pip3 install datasets`") + +try: + from datasets import load_dataset +except: + raise LLMWareException(message="Exception: datasets not found and required for example.") + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +# Run the benchmark test +def run_test(model_name, prompt_list): + print(f"\n > Loading model '{model_name}'") + prompter = Prompt().load_model(model_name) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + for i, entry in enumerate(prompt_list): + + start_time = time.time() + + prompt = entry["query"] + context = entry["context"] + response = prompter.prompt_main(prompt, context=context, prompt_name="default_with_context", temperature=0.3) + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i + 1}. llm_response - {response['llm_response']}") + print(f"{i + 1}. gold_answer - {entry['answer']}") + print(f"{i + 1}. time_taken - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + ds = load_rag_benchmark_tester_dataset() + + # Supported Q4_K_M GGUF Dragon Models: + # -- llmware/dragon-yi-6b-gguf + # -- llmware/dragon-mistral-7b-gguf + # -- llmware/dragon-llama-7b-gguf + + model_name = "llmware/dragon-yi-6b-gguf" + + output = run_test(model_name,ds) diff --git a/solutions/models/dragon_rag_benchmark_tests_huggingface.py b/solutions/models/dragon_rag_benchmark_tests_huggingface.py new file mode 100644 index 0000000..47d989e --- /dev/null +++ b/solutions/models/dragon_rag_benchmark_tests_huggingface.py @@ -0,0 +1,116 @@ + +""" This example demonstrates running a benchmarks set of tests against llmware DRAGON models + https://huggingface.co/collections/llmware/dragon-models-65552d7648093c3f6e35d1bf + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + +import time +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError ("This example requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +# Run the benchmark test +def run_test(model_name, test_dataset): + + # Load the model and tokenizer + print(f"\n > Loading model '{model_name}'") + device = "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype="auto") + else: + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True) + model.to(device) + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + # Run each test + for i, entry in enumerate(test_dataset): + + start_time = time.time() + + # Create and tokenize a prompt + # Note: in our testing, the dragon-yi model performs better with a trailing "\n" at end of prompt + new_prompt = ": " + entry["context"] + "\n" + entry["query"] + "\n" + ":" + "\n" + inputs = tokenizer(new_prompt, return_tensors="pt") + start_of_output = len(inputs.input_ids[0]) + + # Call model.generate() + # Note: temperature: set at 0.3 for consistency of output + # max_new_tokens: set at 100 - may prematurely stop a few of the summaries + outputs = model.generate( + inputs.input_ids.to(device), + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.eos_token_id, + do_sample=True, + temperature=0.3, + max_new_tokens=100, + ) + + output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True) + + # quick/optional post-processing clean-up of potential fine-tuning artifacts + + eot = output_only.find("<|endoftext|>") + if eot > -1: + output_only = output_only[:eot] + + bot = output_only.find(":") + if bot > -1: + output_only = output_only[bot+len(":"):] + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i+1}. llm_response - {output_only}") + print(f"{i+1}. gold_answer - {entry['answer']}") + print(f"{i+1}. time_taken - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + # Get the benchmark dataset + test_dataset = load_rag_benchmark_tester_dataset() + + # BLING MODELS + bling_models = ["llmware/bling-1b-0.1", "llmware/bling-1.4b-0.1", "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0"] + + # DRAGON MODELS + dragon_models = ['llmware/dragon-yi-6b-v0', 'llmware/dragon-red-pajama-7b-v0', 'llmware/dragon-stablelm-7b-v0', + 'llmware/dragon-deci-6b-v0', 'llmware/dragon-mistral-7b-v0','llmware/dragon-falcon-7b-v0', + 'llmware/dragon-llama-7b-v0'] + + # Pick a model: if running on CPU/laptop, select from bling_models list + model_name = dragon_models[0] + output = run_test(model_name,test_dataset) diff --git a/solutions/models/dragon_rag_benchmark_tests_llmware.py b/solutions/models/dragon_rag_benchmark_tests_llmware.py new file mode 100644 index 0000000..7e3eaac --- /dev/null +++ b/solutions/models/dragon_rag_benchmark_tests_llmware.py @@ -0,0 +1,98 @@ + +"""This example demonstrates running a benchmarks set of tests against llmware DRAGON models + https://huggingface.co/collections/llmware/dragon-models-65552d7648093c3f6e35d1bf + The model loading and interaction is handled with the llmware Prompt class which provides additional + capabilities like evidence checking + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + +import time +from llmware.prompts import Prompt + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError ("This example requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + +# Run the benchmark test +def run_test(model_name, prompt_list): + + print(f"\n > Loading model '{model_name}'") + prompter = Prompt().load_model(model_name) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + for i, entry in enumerate(prompt_list): + + start_time = time.time() + + prompt = entry["query"] + context = entry["context"] + response = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context", temperature=0.3) + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i+1}. llm_response - {response['llm_response']}") + print(f"{i+1}. gold_answer - {entry['answer']}") + print(f"{i+1}. time_taken - {time_taken}") + + # Fact checking + fc = prompter.evidence_check_numbers(response) + sc = prompter.evidence_comparison_stats(response) + sr = prompter.evidence_check_sources(response) + for fc_entry in fc: + for f, facts in enumerate(fc_entry["fact_check"]): + print(f"{i+1}. fact_check - {f} {facts}") + + for sc_entry in sc: + print(f"{i+1}. comparison_stats - {sc_entry['comparison_stats']}") + + for sr_entry in sr: + for s, source in enumerate(sr_entry["source_review"]): + print(f"{i+1}. source - {s} {source}") + + return 0 + + +if __name__ == "__main__": + + # Get the benchmark dataset + test_dataset = load_rag_benchmark_tester_dataset() + + # BLING MODELS + bling_models = ["llmware/bling-1b-0.1", "llmware/bling-1.4b-0.1", "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0"] + + # DRAGON MODELS + dragon_models = ['llmware/dragon-yi-6b-v0', 'llmware/dragon-red-pajama-7b-v0', 'llmware/dragon-stablelm-7b-v0', + 'llmware/dragon-deci-6b-v0', 'llmware/dragon-mistral-7b-v0','llmware/dragon-falcon-7b-v0', + 'llmware/dragon-llama-7b-v0'] + + # Pick a model - note: if running on laptop/CPU, select a bling model + model_name = dragon_models[0] + output = run_test(model_name, test_dataset) + diff --git a/solutions/models/fact_checking.py b/solutions/models/fact_checking.py new file mode 100644 index 0000000..1c7fa21 --- /dev/null +++ b/solutions/models/fact_checking.py @@ -0,0 +1,78 @@ + +""" This example demonstrates evidence and fact checking capabilities in the Prompt class. """ + +import os + +from llmware.prompts import Prompt +from llmware.setup import Setup +from llmware.configs import LLMWareConfig +from llmware.dataset_tools import Datasets + + +def contract_analysis_with_fact_checking (model_name): + + # Load the llmware sample files + print (f"\n > Loading the llmware sample files...") + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path,"Agreements") + + print (f"\n > Loading model {model_name}...") + + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + + research = {"topic": "base salary", "prompt": "What is the executive's base salary?"} + + for i, contract in enumerate(os.listdir(contracts_path)): + + print("\nAnalyzing Contract - ", str(i+1), contract) + print("Question: ", research["prompt"]) + + # contract is parsed, text-chunked, and then filtered by "base salary' + source = prompter.add_source_document(contracts_path, contract, query=research["topic"]) + + # calling the LLM with 'source' information from the contract automatically packaged into the prompt + responses = prompter.prompt_with_source(research["prompt"], prompt_name="default_with_context") + + # run several fact checks + ev_numbers = prompter.evidence_check_numbers(responses) + ev_sources = prompter.evidence_check_sources(responses) + ev_stats = prompter.evidence_comparison_stats(responses) + z = prompter.classify_not_found_response(responses, parse_response=True, evidence_match=True,ask_the_model=False) + + for r, response in enumerate(responses): + print("LLM Response: ", response["llm_response"]) + print("Numbers: ", ev_numbers[r]["fact_check"]) + print("Sources: ", ev_sources[r]["source_review"]) + print("Stats: ", ev_stats[r]["comparison_stats"]) + print("Not Found Check: ", z[r]) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + + # Save jsonl report to jsonl to /prompt_history folder + print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + prompter.save_state() + + # Optional - builds a dataset from prompt history that is 'model-training-ready' + ds = Datasets().build_gen_ds_from_prompt_history(prompt_wrapper="human_bot") + + return 0 + + +if __name__ == "__main__": + + hf_model_list = ["llmware/bling-1b-0.1", + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "bling-phi-3-gguf"] + + model_name = hf_model_list[0] + + # to use a 3rd party model, select model_name, e.g., "gpt-4" + # --if model requires an api_key, then it must be set as an os.environ variable + # --see example on 'set_model_api_keys.py' + + contract_analysis_with_fact_checking(model_name) + + diff --git a/solutions/models/huggingface_integration.py b/solutions/models/huggingface_integration.py new file mode 100644 index 0000000..bb5a02d --- /dev/null +++ b/solutions/models/huggingface_integration.py @@ -0,0 +1,213 @@ + +""" This example demonstrates the use of HuggingFace models + 1. Use llmware models available on HuggingFace for generating vector embeddings + 2. Load a basic decoder generative model from Huggingface and use it + 3. Customizing a generative model with weights from a custom fine-tuned model + 4. Using a Transformers model for embedding + 5. Using a SentenceTransformers model for embedding +""" + + +import os +import torch +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query +from llmware.models import ModelCatalog, HFEmbeddingModel +from llmware.prompts import Prompt +from llmware.setup import Setup +from llmware.util import CloudBucketManager + + +# note: starting in llmware-0.1.10, transformers and sentence_transformers are included in the pip install + +try: + from transformers import AutoModel, AutoTokenizer, AutoModelForCausalLM +except ImportError: + raise ImportError ( + "This example requires classes from the 'transformers' Python package. " + "You can install it with 'pip install transformers'" + ) +try: + from sentence_transformers import SentenceTransformer +except ImportError: + raise ImportError ( + "This example requires classes from the 'sentence-transformers' Python package" + "You can install it with 'pip install sentence-transformers'" + ) + + +# Load an llmware model from Hugging Face to generate vector embeddings +def use_llmware_hf_models_for_embedding(): + + # llmware industry models currently published on HuggingFace (more will be coming!) + # *** use any HF embedding model, e.g., BERT, Roberta, etc. + # e.g., llmware_industry_models = "llmware/industry-bert-sec-v0.1", + # "llmware/industry-bert-asset-management-v0.1", + # "llmware/industry-bert-contracts-v0.1", + # "llmware/industry-bert-insurance-v0.1" + + # Choose one + hf_model_name = "llmware/industry-bert-sec-v0.1" + + # Load the model using the Transformer classes and then into llmware using an HFEmbeddingModel + print (f"\n > Loading model '{hf_model_name}'from HuggingFace...") + hf_tokenizer = AutoTokenizer.from_pretrained(hf_model_name) + hf_model = AutoModel.from_pretrained(hf_model_name) + + # pass instantiated HF model and tokenizer to HFEmbeddingModel class + llmware_model = HFEmbeddingModel(model=hf_model, tokenizer=hf_tokenizer,model_name=hf_model_name) + + # Generate an vector embedding + sample = "This is a sample sentence" + vector_embedding = llmware_model.embedding(sample) + print (f"\n > Generating a vector embedding for: '{sample}'\n\n{vector_embedding}") + + return vector_embedding + + +# Load a basic decoder generative model from Huggingface and use it +def load_and_use_decoder_generative_model(): + + # These are some good 'off-the-shelf' smaller testing generative models from HuggingFace + hf_model_testing_list = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b", + "EleutherAI/pythia-70m-v0", "EleutherAI/pythia-160m-v0", "EleutherAI/pythia-410m-v0", + "EleutherAI/pythia-1b-v0", "EleutherAI/pythia-1.4b-v0"] + + # Here we'll just select one of the above models + model_name = hf_model_testing_list[6] + + # Load the model using the Transformer classes + print (f"\n > Loading model '{model_name}'from HuggingFace...") + hf_model = AutoModelForCausalLM.from_pretrained(model_name) + hf_tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Bring the model into llware. These models were not trained on instruction following, + # so we set instruction_following to False + model = ModelCatalog().load_hf_generative_model(hf_model, hf_tokenizer, instruction_following=False) + + # Make a call to the model + prompt_text = "The future of artificial intelligence is likely to be" + print (f"\n > Prompting the model with '{prompt_text}'") + output = model.inference(prompt_text)["llm_response"] + print(f"\nResponse:\n{prompt_text}{output}") + + return output + + +# Load a HuggingFace generative model and override the weights to use a custom user-developed fine-tuned model +def override_generative_model_weights_with_custom_fine_tuned_model(): + + # These are some good 'off-the-shelf' smaller testing generative models from HuggingFace + hf_model_testing_list = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b", + "EleutherAI/pythia-70m-v0", "EleutherAI/pythia-160m-v0", "EleutherAI/pythia-410m-v0", + "EleutherAI/pythia-1b-v0", "EleutherAI/pythia-1.4b-v0"] + + # Select a model + model_name = "EleutherAI/pythia-410m-v0" + + # Load the model using the Transformer classes + print (f"\n > Loading model '{model_name}'from HuggingFace...") + hf_model = AutoModelForCausalLM.from_pretrained(model_name) + hf_tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Retrive the custom fine-tuned model + # Note: This is a custom model that has been developed only for testing and demonstration purposes + custom_model_name = "contracts-pythia-hf-410m-v0" + print (f"\n > Loading custom model '{custom_model_name}'from llmware...") + custom_model_path = os.path.join(LLMWareConfig.get_model_repo_path(),custom_model_name) + if not os.path.exists(custom_model_path): + CloudBucketManager().pull_single_model_from_llmware_public_repo(custom_model_name) + + # Override the hf_model default model weights with our own custom-trained weights and load it into llmware + print (f"\n > Overriding model '{model_name}' to use custom-trained weights from '{custom_model_name}'...") + hf_model.load_state_dict(torch.load(os.path.join(custom_model_path,"pytorch_model.bin"), map_location=torch.device('cpu')), strict=False) + model = ModelCatalog().load_hf_generative_model(hf_model, hf_tokenizer, instruction_following=False) + + # Interact with the model + prompt_text = "According to the terms of the executive stock option plan," + print (f"\n > Prompting the model with '{prompt_text}'") + output = model.inference(prompt_text)["llm_response"] + print(f"\nResponse:\n{prompt_text}{output}") + + return output + + +# Use a Transformers model for embedding +def use_transformers_model_for_embedding(library_name, model_name): + + # Create a library and add some documents so we can do some vector embeddings + print (f"\n > Creating a library...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "SmallLibrary")) + + # Load the model + print (f"\n > Loading model '{model_name}'") + hf_model = AutoModel.from_pretrained(model_name) + hf_tokenizer = AutoTokenizer.from_pretrained(model_name) + + # Create vector embeddings + print (f"\n > Creating vector embeddings...") + library.install_new_embedding(model=hf_model, tokenizer=hf_tokenizer, from_hf=True, vector_db="faiss", batch_size=50) + + # Perform a query + query_term = "salary" + print (f"\n > Performing query for {query_term}...") + query = Query(library=library, embedding_model_name=model_name, embedding_model=hf_model, tokenizer=hf_tokenizer, from_hf=True) + query_results = query.semantic_query(query_term,result_count=3) + print (f"Top 3 Results:") + for i, result in enumerate(query_results): + file_source = result["file_source"] + page_num = result["page_num"] + text = result["text"] + print(f"\n - From {file_source} (page {page_num}):\n{text}") + + return 0 + + +# Use a SentenceTransformers model for embedding +def use_sentence_transformers_model_for_embedding(library_name, model_name): + + # Create a library and add some documents so we can do some vector embeddings + print (f"\n > Creating a library...") + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(input_folder_path=os.path.join(sample_files_path, "SmallLibrary")) + + # Load the model + print (f"\n > Loading model '{model_name}'") + sbert_model = SentenceTransformer(model_name) + + # Create vector embeddings + print (f"\n > Creating vector embeddings...") + library.install_new_embedding(model=sbert_model, embedding_model_name=model_name, + from_sentence_transformer=True, vector_db="faiss", batch_size=100) + + # Perform a query + query_term = "salary" + print (f"\n > Performing query for {query_term}...") + + query= Query(library=library, + embedding_model_name=model_name, + embedding_model=sbert_model, + from_sentence_transformer=True) + + query_results = query.semantic_query(query_term, result_count=3) + print (f"Top 3 Results:") + for i, result in enumerate(query_results): + file_source = result["file_source"] + page_num = result["page_num"] + text = result["text"] + print(f"\n - From {file_source} (page {page_num}):\n{text}") + + return query_results + + +if __name__ == "__main__": + + use_llmware_hf_models_for_embedding() + load_and_use_decoder_generative_model() + override_generative_model_weights_with_custom_fine_tuned_model() + use_transformers_model_for_embedding("test_transformers", "bert-base-cased") + use_sentence_transformers_model_for_embedding("test_sentence_transformers", "all-distilroberta-v1") diff --git a/solutions/models/llmware_model_fast_start.py b/solutions/models/llmware_model_fast_start.py new file mode 100644 index 0000000..fed57fc --- /dev/null +++ b/solutions/models/llmware_model_fast_start.py @@ -0,0 +1,166 @@ + +"""This example demonstrates running a benchmarks set of tests against any llmware model in HuggingFace + https://huggingface.co/llmware + + Usage: You can pass in a model name: + python llmware_model_fast_start.py llmware/bling-1b-0.1 + If you do not specify a model you will be prompted to pick one + + This example uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on + Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the + datasets library, which can be installed with: + + `pip3 install datasets` + +""" + +import re +import sys +import time +import torch +from huggingface_hub import hf_api, ModelCard +from transformers import AutoModelForCausalLM, AutoTokenizer + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError ("This example requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + + +# Query HuggingFace and get the llmware models. Return the the components of a table: headers and data +def get_llmware_models(): + table_headers=['','MODEL','DETAILS'] + table_data=[] + + models = hf_api.list_models(author="llmware") + sorted_models = sorted(models, key=lambda x: x.id) + + for i, model in enumerate(sorted_models): + model_card_content = ModelCard.load(model.id).content + match = re.search(r"Model type:\*\* (.+?)\n", model_card_content) # Get type from a line like this: - **Model type:** GPTNeoX instruct-trained decoder + model_type = "" + if match: + model_type = match.group(1).strip() + model_details = f"{model_type} ({model.downloads} downloads)" + table_data.append([i+1, model.id, model_details]) + + return table_headers, table_data + + +def print_llmware_models(): + + table_headers, table_data = get_llmware_models() + + print(table_headers[0], "\t\t", table_headers[1], "\t\t", table_headers[2]) + for row in table_data: + print(row[0], "\t\t", row[1], "\t\t", row[2]) + + +def prompt_user_for_model_selection(prompt=None): + + table_headers, table_data = get_llmware_models() + + print(table_headers[0], "\t\t", table_headers[1], "\t\t", table_headers[2]) + for row in table_data: + print(row[0], "\t\t", row[1], "\t\t", row[2]) + + num_models = len(table_data) + + if prompt is None: + prompt = f"\nSelect a model (1-{num_models}): " + while True: + try: + user_input = input(prompt) + user_integer = int(user_input) + if user_integer not in range(1,num_models+1): + continue + return table_data[user_integer-1][1] + except ValueError: + print("That's not an integer. Please try again.") + return None + + +# Pull a 200 question RAG benchmark test dataset from llmware HuggingFace repo +def load_rag_benchmark_tester_dataset(): + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +# Run the benchmark test +def run_test(model_name, test_dataset): + + # Load the model and tokenizer + print(f"\n > Loading model '{model_name}'") + device = "cuda" if torch.cuda.is_available() else "cpu" + if torch.cuda.is_available(): + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, torch_dtype="auto") + else: + model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True) + model.to(device) + tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) + + print(f"\n > Running RAG Benchmark Test against '{model_name}' - 200 questions") + # Run each test + for i, entry in enumerate(test_dataset): + + start_time = time.time() + + # Create and tokenize a prompt + # Note: in our testing, the dragon-yi model performs better with a trailing "\n" at end of prompt + new_prompt = ": " + entry["context"] + "\n" + entry["query"] + "\n" + ":" + "\n" + inputs = tokenizer(new_prompt, return_tensors="pt") + start_of_output = len(inputs.input_ids[0]) + + # Call model.generate() + # Note: temperature: set at 0.3 for consistency of output + # max_new_tokens: set at 100 - may prematurely stop a few of the summaries + outputs = model.generate( + inputs.input_ids.to(device), + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.eos_token_id, + do_sample=True, + temperature=0.3, + max_new_tokens=100, + ) + + output_only = tokenizer.decode(outputs[0][start_of_output:],skip_special_tokens=True) + + # quick/optional post-processing clean-up of potential fine-tuning artifacts + + eot = output_only.find("<|endoftext|>") + if eot > -1: + output_only = output_only[:eot] + + bot = output_only.find(":") + if bot > -1: + output_only = output_only[bot+len(":"):] + + # Print results + time_taken = round(time.time() - start_time, 2) + print("\n") + print(f"{i+1}. llm_response - {output_only}") + print(f"{i+1}. gold_answer - {entry['answer']}") + print(f"{i+1}. time_taken - {time_taken}") + + return 0 + + +if __name__ == "__main__": + + # Prompt user to get model if not passed in as an argument + if len(sys.argv) > 1: + selected_model = sys.argv[1] + else: + selected_model = prompt_user_for_model_selection() + test_dataset = load_rag_benchmark_tester_dataset() + output = run_test(selected_model,test_dataset) diff --git a/solutions/models/model_catalog_registry.py b/solutions/models/model_catalog_registry.py new file mode 100644 index 0000000..650ef4a --- /dev/null +++ b/solutions/models/model_catalog_registry.py @@ -0,0 +1,55 @@ + +"""This example illustrates the basic capabilities of the Model Catalog and how to use""" + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + +# 1 - to list all of the models registered by default in llmware +mc = ModelCatalog().list_all_models() + +print("\nAll-Models-List") +for i, models in enumerate(mc): + print("update: models - ", i, models) + + +# 2 - to list all generative local models +gl_mc = ModelCatalog().list_generative_local_models() + +print("\nGenerative-Local-Models-List") +for i, models in enumerate(gl_mc): + print("update: gen local models - ", i, models) + + +# 3 - to add models to the catalog, e.g., sentence transformer + +ModelCatalog().register_sentence_transformer_model("all-MiniLM-L6-v2", embedding_dims=384, + context_window=256) + +""" +ModelRegistry().add_model_list({"model_name": "all-MiniLM-L6-v2", "model_category": "embedding", + "embedding_dims":384, "context_window":256, "model_family": "LLMWareSemanticModel", + "display_name": "MiniLM", "model_location": "st_repo"}) +""" + +# to confirm that model was added +model_card = ModelCatalog().lookup_model_card("all-MiniLM-L6-v2") +print("\nupdate: Registered new embedding model - ", model_card) + +# 4 - embedding models +emb_models = ModelCatalog().list_embedding_models() + +# note: newly created 'all-MiniLM-L6-v2' will now be included +print("\nEmbedding-Models-List") +for i, models in enumerate(emb_models): + print("update: embedding models - ", i, models) + +# 5 - to delete model from catalog +ModelCatalog().delete_model_card("gpt-3.5-turbo") + +# 6 - to load any model in the catalog +new_model = ModelCatalog().load_model("all-MiniLM-L6-v2") + +# 7 - equivalent 'load_model' with any prompt +# -- note: if this model is not installed locally, this will pull it down into local cache +prompter = Prompt().load_model("llmware/bling-1b-0.1") + diff --git a/solutions/models/prompt_state.py b/solutions/models/prompt_state.py new file mode 100644 index 0000000..c3a2c91 --- /dev/null +++ b/solutions/models/prompt_state.py @@ -0,0 +1,78 @@ + +"""This example demonstrates basic use of Prompts, and how to capture and track the Prompt State and +interaction history.""" + + +from llmware.prompts import Prompt +from llmware.resources import PromptState + +def prompt_state(llm_model): + + # Create a new prompter with state persistence + prompter = Prompt(save_state=True) + + # Capture the prompt_id (which can be used later to reload state) + prompt_id = prompter.prompt_id + + # Load the model + prompter.load_model(llm_model, temperature=0.0, sample=False) + + # Define a list of prompts + prompts = [ + {"query": "How old is Bob?", "context": "John is 43 years old. Bob is 27 years old."}, + {"query": "When did COVID start?", "context": "COVID started in March of 2020 in most of the world."}, + {"query": "What is the current stock price?", "context": "The stock is trading at $26 today."}, + {"query": "When is the big game?", "context": "The big game will be played on November 14, 2023."}, + {"query": "What is the CFO's salary?", "context": "The CFO has a salary of $285,000."}, + {"query": "What grade is Michael in school?", "context": "Michael is starting 11th grade."} + ] + + # Iterate through the prompt which will save each response dict in in the prompt_state + print (f"> Sending a series of prompts to {llm_model}...") + + for i, prompt in enumerate(prompts): + print (" - " + prompt["query"]) + response = prompter.prompt_main(prompt["query"] ,context=prompt["context"] ,register_trx=True) + + print(f" - LLM Responses: {response}") + + # Print how many interactions are now in the prompt history + interaction_history = prompter.interaction_history + print (f"> Prompt Interaction History now contains {len(interaction_history)} interactions") + + # Use the dialog_tracker to regenerate the conversation with the LLM + print (f"> Reconstructed Dialog") + dialog_history = prompter.dialog_tracker + for i, conversation_turn in enumerate(dialog_history): + print(" - ", i, "[user]: ", conversation_turn["user"]) + print(" - ", i, "[ bot]: ", conversation_turn["bot"]) + + # Saving and clean the prompt state + prompter.save_state() + prompter.clear_history() + + # Print the number of interactions + interaction_history = prompter.interaction_history + print (f"> Prompt history has been cleared") + print (f"> Prompt Interaction History now contains {len(interaction_history)} interactions") + + # Reload the prompt state using the prompt_id and print again the number of interactions + prompter.load_state(prompt_id) + interaction_history = prompter.interaction_history + print (f"> The previous prompt state has been re-loaded") + print (f"> Prompt Interaction History now contains {len(interaction_history)} interactions") + + # Generate a Prompt transaction report + prompt_transaction_report = PromptState().generate_interaction_report([prompt_id]) + print (f"> A prompt transaction report has been generated: {prompt_transaction_report}") + + return 0 + + +if __name__ == "__main__": + + model_name = "llmware/bling-1b-0.1" + + print(f"\nExample - basic prompt state and interaction history management.\n") + + prompt_state(model_name) diff --git a/solutions/models/prompt_with_pdf_source.py b/solutions/models/prompt_with_pdf_source.py new file mode 100644 index 0000000..fb045d2 --- /dev/null +++ b/solutions/models/prompt_with_pdf_source.py @@ -0,0 +1,48 @@ + +"""This example demonstrates how to parse a PDF document 'in-line' and integrate in memory directly into a +Prompt as a source of evidence for an LLM inference. """ + +import os +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def prompt_with_sources (model_name): + + # pulls down the sample files, including a specific agreement file + sample_files_path = Setup().load_sample_files(over_write=False) + fp = os.path.join(sample_files_path, "Agreements") + + local_file = "Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + + prompter = Prompt().load_model(model_name) + + # .add_source_document will do the following: + # 1. parse the file (any supported document type) + # 2. apply an optional query filter to reduce the text chunks to only those matching the query + # 3. batch according to the model context window, and make available for any future inferences + + sources = prompter.add_source_document(fp, local_file, query="base salary") + + prompt = "What is the base salary amount?" + prompt_instruction = "default_with_context" + response = prompter.prompt_with_source(prompt=prompt, prompt_name=prompt_instruction) + + print(f"LLM Response - {response}") + + response_display = response[0]["llm_response"] + print (f"- Context: {local_file}\n- Prompt: {prompt}\n- LLM Response:\n{response_display}") + + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + model_name = "llmware/bling-1b-0.1" + + print(f"\nExample - intro to prompt_with_sources - adding a document source to a prompt\n") + + prompt_with_sources (model_name) + diff --git a/solutions/models/prompt_with_sources.py b/solutions/models/prompt_with_sources.py new file mode 100644 index 0000000..4b6840e --- /dev/null +++ b/solutions/models/prompt_with_sources.py @@ -0,0 +1,82 @@ + +""""This example demonstrates: + prompt_with_sources - powerful abstraction to integrate various knowledge sources into a prompt +""" + + +import os +from llmware.prompts import Prompt +from llmware.setup import Setup +from llmware.models import PromptCatalog +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + + +def prompt_with_sources(model_name, library_name): + + print(f"Example - prompt_with_sources - attaching several different knowledge sources to a Prompt directly.") + + library = Library().create_new_library(library_name) + + sample_files_path = Setup().load_sample_files(over_write=False) + + ingestion_folder_path = os.path.join(sample_files_path, "Agreements") + parsing_output = library.add_files(ingestion_folder_path) + + local_file = "Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + + prompter = Prompt().load_model(model_name) + + # Use #1 - add_source_document - parses the document in memory, filters the text chunks by query, and then + # creates a 'source' context to be passed to the model + + print(f"\n#1 - add a source document file directly into a prompt") + + sources2 = prompter.add_source_document(ingestion_folder_path, local_file, query="base salary") + + prompt = "What is the base salary amount?" + prompt_instruction="default_with_context" + response = prompter.prompt_with_source(prompt=prompt, prompt_name=prompt_instruction)[0]["llm_response"] + print (f"- Context: {local_file}\n- Prompt: {prompt}\n- LLM Response:\n{response}") + prompter.clear_source_materials() + + # Use #2 - add_source_wikipedia - gets a source document from Wikipedia on Barack Obama, + # and creates source context + + print(f"\n#2 - add a wikipedia article by api call by topic into a prompt") + + prompt = "Was Barack Obama the Prime Minister of Canada?" + wiki_topic = "Barack Obama" + prompt_instruction = "yes_no" + sources3 = prompter.add_source_wikipedia(wiki_topic, article_count=1) + response = prompter.prompt_with_source(prompt=prompt, prompt_name=prompt_instruction)[0]["llm_response"] + print (f"- Context: {wiki_topic}\n- Prompt: {prompt}\n- LLM Response:\n{response}") + prompter.clear_source_materials() + + # Use #3 - add_source_query_results - produces the same results as the first case, but runs a query on the library + # and then adds the query results to the prompt which are concatenated into a source context + + print(f"\n#3 - run a query on a library and then pass the query results into a prompt") + + query_results = Query(library).text_query("base salary") + prompt = "What is the annual rate of the base salary?" + sources4 = prompter.add_source_query_results(query_results) + response = prompter.prompt_with_source(prompt=prompt, prompt_name=prompt_instruction)[0]["llm_response"] + print(f"- Context: {local_file}\n- Prompt: {prompt}\n- LLM Response:\n{response}") + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + # this model is a placeholder which will run on local laptop - swap out for higher accuracy, larger models + model_name = "llmware/bling-1b-0.1" + + library_name = "lib_prompt_with_sources_1" + + prompt_with_sources(model_name,library_name) + diff --git a/solutions/models/set_model_api_keys.py b/solutions/models/set_model_api_keys.py new file mode 100644 index 0000000..640e3dc --- /dev/null +++ b/solutions/models/set_model_api_keys.py @@ -0,0 +1,40 @@ + +"""This example script shows how to set the os.environ variables with API keys for supported models""" + +import os + + +def user_managed_secrets_setup(key, value): + + user_managed_access_key_list = ["USER_MANAGED_OPENAI_API_KEY", + "USER_MANAGED_COHERE_API_KEY", + "USER_MANAGED_ANTHROPIC_API_KEY", + "USER_MANAGED_AI21_API_KEY", + "USER_MANAGED_GOOGLE_API_KEY", + "USER_MANAGED_PINECONE_API_KEY", + "USER_MANAGED_PINECONE_ENVIRONMENT", + "USER_MANAGED_AWS_ACCESS_KEY", + "USER_MANAGED_AWS_SECRET_KEY"] + + if key in user_managed_access_key_list: + os.environ[key] = value + + # set environ variables- and will be automatically passed to corresponding model when invoked + + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "{INSERT-YOUR-OPENAI-KEY}" + # os.environ["USER_MANAGED_COHERE_API_KEY"] = "{INSERT_YOUR-COHERE-API-KEY}" + # os.environ["USER_MANAGED_ANTHROPIC_API_KEY"] = "{INSERT_YOUR_ANTHROPIC_API_KEY}" + # os.environ["USER_MANAGED_AI21_API_KEY"] = "{INSERT_YOUR_AI21_API_KEY}" + # os.environ["USER_MANAGED_GOOGLE_API_KEY"] = "{INSERT_YOUR_GOOGLE_API_KEY}" + # os.environ["USER_MANAGED_PINECONE_API_KEY"] = "{INSERT_YOUR_PINECONE_API_KEY}" + # os.environ["USER_MANAGED_PINECONE_ENVIRONMENT"] = "{INSERT_YOUR_PINECONE_ENVIRONMENT_KEY}" + # os.environ["USER_MANAGED_AWS_ACCESS_KEY"] = "{INSERT_YOUR_AWS_ACCESS_KEY}" + # os.environ["USER_MANAGED_AWS_SECRET_KEY"] = "{INSERT_YOUR_AWS_SECRET_KEY}" + + return 0 + +# Example Usage +# os.environ["USER_MANAGED_OPENAI_API_KEY"] = "..." +# prompter = Prompt().load_model("gpt-4") +# --> prompter will look to the environ variable and pass the API key + diff --git a/solutions/models/using-azure-openai.py b/solutions/models/using-azure-openai.py new file mode 100644 index 0000000..2d31215 --- /dev/null +++ b/solutions/models/using-azure-openai.py @@ -0,0 +1,72 @@ + +""" This example shows how to use OpenAIConfigs to create a configured OpenAI client, most often used for +Azure OpenAI access.""" + +import os + +from llmware.models import ModelCatalog +from llmware.configs import OpenAIConfig +from openai import AzureOpenAI + + +# Set the following environment variables: +# - AZURE_OPENAI_ENDPOINT : found on your Azure OpenAI page +# - AZURE_OPENAI_API_KEY : found on your Azure OpenAI page +# - USER_MANAGED_OPENAI_API_KEY : found on you OpenAI API page +# +# Additionally, with this example, you will need an Azure OpenAI deployment +# for gpt-4 and text-embedding-3-small, but feel free to replace these below. +# +# Make sure to replace the deployment names with your deployments in the +# AzureOpenAI clients created below. + + +# to start - OpenAI client is created in OpenAI Generative and Embedding models classes at the time of inference +# the client will be created as a standard OpenAI client with the api_keys passed + +my_azure_client = OpenAIConfig().get_azure_client() +print("my azure client to start: ", my_azure_client) + +# to configure an AzureOpenAI client, two steps: +# first, create the client with openai >= 1.0 python SDK, (see above) e.g.: + +gpt4_client = AzureOpenAI( + azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT"), + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + api_version="2024-02-01", + azure_deployment="your-gpt-4-deployment-name" +) + +# second, set the azure client in OpenAIConfigs as below: +OpenAIConfig().set_azure_client(gpt4_client) +print("my azure client - set: ", OpenAIConfig().get_azure_client()) + +# now, run the inference like any other in llmware + +# OpenAI Generative call +model = ModelCatalog().load_model("gpt-4") + +# the model will check the value of get_azure_client() in the configs -> if set, then will use +response = model.inference("What is the future of AI") +print("response: ", response) + +text_embedding_client = AzureOpenAI( + azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT"), + api_key=os.getenv("AZURE_OPENAI_API_KEY"), + api_version="2024-02-01", + azure_deployment="your-text-embedding-3-small-deployment-name" +) + +OpenAIConfig().set_azure_client(text_embedding_client) + +# OpenAI Embedding call +model = ModelCatalog().load_model("text-embedding-3-small") +embedding = model.embedding(["This is a sample sentence for an embedding test."]) +print("embedding: ", embedding) + +# reset so you can use the standard OpenAI client +OpenAIConfig().set_azure_client(None) + +model = ModelCatalog().load_model("text-embedding-3-small", api_key=os.getenv("USER_MANAGED_OPENAI_API_KEY")) +embedding = model.embedding(["This is a sample sentence for an embedding test."]) +print("embedding: ", embedding) diff --git a/solutions/models/using-llama-3.py b/solutions/models/using-llama-3.py new file mode 100644 index 0000000..358d2a0 --- /dev/null +++ b/solutions/models/using-llama-3.py @@ -0,0 +1,31 @@ + +""" This example shows how to use the new Llama-3 Model with llmware, as well as how to access quantized versions. """ + +from llmware.models import ModelCatalog + +# *** CORE PYTORCH LLAMA-3 MODELS *** + +# llama-3-8b models pre-registered in the model catalog: +# llama-3-base - "Meta-Llama-3-8B-Instruct" or "llama-3-instruct" +# llama-3-instruct - "Meta-Llama-3-8B" or "llama-3-base" + +# note: to access these models in llmware requires two pre-registration steps: +# 1. meta-llama registration - https://llama.meta.com/docs/get-started/ - requires accepting the llama-3 licensing terms +# 2. huggingface api key (does not require any payment, but you need a free HF account), +# e.g., hf_key = "hf_...." + +# once you have completed these steps, you can access in llmware as follows: +# # llama3_model = ModelCatalog().load_model(selected_llama_model) + +# *** LLAMA-3 GGUF MODELS *** + +# 3 quantized models added to the default Model Catalog: +# model_name = "bartowski/Meta-Llama-3-8B-Instruct-GGUF" +# model_name = "QuantFactory/Meta-Llama-3-8B-Instruct-GGUF" +# model_name = "QuantFactory/Meta-Llama-3-8B-GGUF" + +l3_gguf = ModelCatalog().load_model("bartowski/Meta-Llama-3-8B-Instruct-GGUF") + +response = l3_gguf.inference("I am going to Mumbai. What should I see?") +print("\nllama3-gguf response: ", response) + diff --git a/solutions/models/using-microsoft-phi-3.py b/solutions/models/using-microsoft-phi-3.py new file mode 100644 index 0000000..557ba89 --- /dev/null +++ b/solutions/models/using-microsoft-phi-3.py @@ -0,0 +1,30 @@ + +""" This example shows how to use the new Microsoft Phi-3 model. """ + +from llmware.models import ModelCatalog + +# phi-3 models pre-registered in the model catalog (as of Tues, April 23 when model launched): +# phi-3 - "microsoft/Phi-3-mini-4k-instruct" +# phi-3-128k - "microsoft/Phi-3-mini-128k-instruct" +# phi-3-gguf - "microsoft/Phi-3-mini-4k-instruct-gguf" + +# first let's try the pytorch version +# note: if not running on a cuda machine, you may see warnings about flash_attn not present +# ... and it will be a little slow to load + +phi3 = ModelCatalog().load_model("phi-3") # use "phi-3-128k" for the 128k context +response = phi3.inference("I am going to Mumbai. What should I see?") +print("\nresponse: ", response) + +# second, use the gguf version +phi3_gguf = ModelCatalog().load_model("phi-3-gguf") + +response = phi3_gguf.inference("I am going to Mumbai. What should I see?") +print("\ngguf response: ", response) + +# now, try with a context sample +context = "The stock is now soaring to $120 per share after great earnings." +response = phi3_gguf.inference("What is the current stock price?", add_context=context) + +print("\ngguf response: ", response) + diff --git a/solutions/models/using-ollama-models.py b/solutions/models/using-ollama-models.py new file mode 100644 index 0000000..d0d4588 --- /dev/null +++ b/solutions/models/using-ollama-models.py @@ -0,0 +1,81 @@ + +""" This example illustrates how to use Ollama models in llmware. It assumes that you have separately + downloaded and installed Ollama and used 'ollama run {model_name}' to cache several models in + ollama. """ + +from llmware.models import ModelCatalog + +# Step 1 - register your Ollama models in llmware ModelCatalog +# -- these two lines will register: llama2 and mistral models +# -- note: assumes that you have previously cached and installed both of these models with ollama locally + +# register llama2 +ModelCatalog().register_ollama_model(model_name="llama2",model_type="chat",host="localhost",port=11434) + +# register mistral - note: if you are using ollama defaults, then OK to register with ollama model name only +ModelCatalog().register_ollama_model(model_name="mistral") + +# optional - confirm that model was registered +my_new_model_card = ModelCatalog().lookup_model_card("llama2") +print("\nupdate: confirming - new ollama model card - ", my_new_model_card) + +# Step 2 - start using the Ollama model like any other model in llmware + +print("\nupdate: calling ollama llama 2 model ...") + +model = ModelCatalog().load_model("llama2") +response = model.inference("why is the sky blue?") + +print("update: example #1 - ollama llama 2 response - ", response) + +# Tip: if you are loading 'llama2' chat model from Ollama, note that it is already included in +# the llmware model catalog under a different name, "TheBloke/Llama-2-7B-Chat-GGUF" +# the llmware model name maps to the original HuggingFace repository, and is a nod to "TheBloke" who has +# led the popularization of GGUF - and is responsible for creating most of the GGUF model versions. +# --llmware uses the "Q4_K_M" model by default, while Ollama generally prefers "Q4_0" + +print("\nupdate: calling Llama-2-7B-Chat-GGUF in llmware catalog ...") + +model = ModelCatalog().load_model("TheBloke/Llama-2-7B-Chat-GGUF") +response = model.inference("why is the sky blue?") + +print("update: example #1 - [compare] - llmware / Llama-2-7B-Chat-GGUF response - ", response) + +# Now, let's try the Ollama Mistral model with a context passage + +model2 = ModelCatalog().load_model("mistral") + +context_passage= ("NASA’s rover Perseverance has gathered data confirming the existence of ancient lake " + "sediments deposited by water that once filled a giant basin on Mars called Jerezo Crater, " + "according to a study published on Friday. The findings from ground-penetrating radar " + "observations conducted by the robotic rover substantiate previous orbital imagery and " + "other data leading scientists to theorize that portions of Mars were once covered in water " + "and may have harbored microbial life. The research, led by teams from the University of " + "California at Los Angeles (UCLA) and the University of Oslo, was published in the " + "journal Science Advances. It was based on subsurface scans taken by the car-sized, six-wheeled " + "rover over several months of 2022 as it made its way across the Martian surface from the " + "crater floor onto an adjacent expanse of braided, sedimentary-like features resembling, " + "from orbit, the river deltas found on Earth.") + +response = model2.inference("What are the top 3 points?", add_context=context_passage) + +print("\nupdate: calling ollama mistral model ...") + +print("update: example #2 - ollama mistral response - ", response) + +# Step 3 - using the ollama discovery API - optional + +discovery = model2.discover_models() +print("\nupdate: example #3 - checking ollama model manifest list: ", discovery) + +if len(discovery) > 0: + # note: assumes tht you have at least one model registered in ollama -otherwise, may throw error + for i, models in enumerate(discovery["models"]): + print("ollama models: ", i, models) + + +# for more information and other alternatives for using GGUF models, please see the following examples: +# -- examples/Models/chat_gguf_fast_start.py +# -- examples/Models/using_gguf.py +# -- examples/Models/using-open-chat-models.py +# -- examples/Models/dragon-gguf_fast_start.py diff --git a/solutions/models/using-open-chat-models.py b/solutions/models/using-open-chat-models.py new file mode 100644 index 0000000..b4b6c75 --- /dev/null +++ b/solutions/models/using-open-chat-models.py @@ -0,0 +1,60 @@ + +""" + This example shows how to use 'Open Chat' inference models that expose an endpoint compatible with the + OpenAI API - using 'api_base' to configure the endpoint uri + + For example, to integrate a model on LM Studio with standard configuration: + -- api_base = 'http://localhost:1234/v1' + + Please also note that llmware implements llama.cpp directly, so you can run inference on any GGUF models + very easily and natively in llmware - see the GGUF example in /Models/using_gguf.py' +""" + + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + + +# one step process: add the open chat model to the Model Registry +# key params: +# model_name = "my_open_chat_model1" +# api_base = uri_path to the proposed endpoint +# prompt_wrapper = alpaca | | chat_ml | hf_chat | human_bot +# -> Llama2-Chat +# hf_chat -> Zephyr-Mistral +# chat_ml -> OpenHermes - Mistral +# human_bot -> Dragon models +# model_type = "chat" (alternative: "completion") + +ModelCatalog().register_open_chat_model("my_open_chat_model1", + api_base="http://localhost:1234/v1", + prompt_wrapper="", + model_type="chat") + +# once registered, you can invoke like any other model in llmware + +prompter = Prompt().load_model("my_open_chat_model1") +response = prompter.prompt_main("What is the future of AI?") + + +# you can (optionally) register multiple open chat models with different api_base and model attributes + +ModelCatalog().register_open_chat_model("my_open_chat_model2", + api_base="http://localhost:5678/v1", + prompt_wrapper="hf_chat", + model_type="chat") + + +# you can also alternate with open ai models - which will 'revert' to the default openai api_base + +openai_prompter = Prompt().load_model("gpt-3.5.-turbo-instruct") + + +# if you list all of the models in the catalog, you will see the two newly created open chat models + +my_models = ModelCatalog().list_all_models() + +for i, mods in enumerate(my_models): + print("models: ", i, mods) + + diff --git a/solutions/models/using-phi-3-function-calls.py b/solutions/models/using-phi-3-function-calls.py new file mode 100644 index 0000000..f7f100c --- /dev/null +++ b/solutions/models/using-phi-3-function-calls.py @@ -0,0 +1,99 @@ + +""" This example shows how to use 7 different SLIM function calling models fine-tuned on top of Phi-3: + + -- Extraction - slim-extract-phi-3-gguf - generates python dictionary with 'key' and 'value' + -- Summarization - slim-summary-phi-3-gguf - generates python list with key bullet-point summary + -- XSUM (titles) - slim-xsum-phi-3-gguf - generates python dictionary with 'xsum' key + -- Boolean - slim-boolean-phi-3-gguf - generate python dictionary with "answer" & "explanation" keys + -- Sentiment-NER - slim-sa-ner-phi-3-gguf - generates python dictionary with "sentiment" and selected ner keys + -- Question-Gen - slim-q-gen-phi-3-tool - generates python dictionary with "question" key + -- Question-Answer - slim-qa-gen-phi-3-tool - generates python dictionary with "question" and "answer" key + + The design of these models is to simplify both the input prompt and output to enable easy integration into + programmatic workflows. + + """ + +from llmware.models import ModelCatalog + +# sample text passage that will be used as the basis for the function call analysis + +context_passage = ("Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter.") + + +# for convenience to execute a 'loop', we will set up a dictionary with each function call, and the associated +# model and parameters that are being passed to the model + +phi3_function_call_models = { + + # extract model will look for the 'key' in the params, and return the 'value' found in the text + "extract": {"model": "slim-extract-phi-3-gguf", "params": ["net income"]}, + + # summary model will return a python list with key summary points related to the parameter + "summary": {"model": "slim-summary-phi-3-gguf", "params": ["financial highlights"]}, + + # xsum model produces an 'extreme summarization', e.g. a headline or title + "xsum": {"model": "slim-xsum-phi-3-gguf", "params": ["xsum"]}, + + # boolean model is designed to answer yes/no questions + "boolean": {"model": "slim-boolean-phi-3-gguf", "params": ["Is Best Buy closing stores? (explain)"]}, + + # sentiment-ner model returns several keys for sentiment and ner attributes (e.g., people, place, organization) + "sentiment-ner": {"model": "slim-sa-ner-phi-3-gguf", "params": ["sentiment", "people"]}, + + # q-gen model generates a question from the context passage + "q-gen": {"model": "slim-q-gen-phi-3-tool", "params": ["question"]}, + + # qa-gen model generates question and answer from the context passage + "qa-gen": {"model": "slim-qa-gen-phi-3-tool", "params": ["question, answer"]} + } + +for function, model in phi3_function_call_models.items(): + + print(f"\nfunction: {function} - model - {model['model']} - params - {model['params']}") + slim_model = ModelCatalog().load_model(model["model"], temperature=0.0, sample=False) + + # note: this is the line doing all of the work - each model has been fine-tuned as a 'specialist' for + # its function, so the only required inputs are the source context passage, and the specific parameters to be used + + response = slim_model.function_call(context_passage, params=model["params"]) + + print("response: ", response) + + + diff --git a/solutions/models/using-qwen2-models.py b/solutions/models/using-qwen2-models.py new file mode 100644 index 0000000..c3c2334 --- /dev/null +++ b/solutions/models/using-qwen2-models.py @@ -0,0 +1,57 @@ + +""" This example shows how to use Qwen2 models in LLMWare, consisting of three main categories - + + 1 - standard QWEN2 chat/instruct models, packaged in GGUF in 7B / 1.5B / 0.5B sizes. + + 2 - RAG fine-tuned QWEN2 in DRAGON and BLING series. + + 3 - Extract function-calling finetune in SLIM series. + +""" + +from llmware.models import ModelCatalog + +# 1 - MAIN CATALOG - 3 QWEN2 GGUF models for chat (7B / 1.5B / 0.5B) + +qwen2_base_gguf = ["qwen2-7b-instruct-gguf", "qwen2-1.5b-instruct-gguf", "qwen2-0.5b-instruct-gguf"] + +print("\nExample #1 - loading Qwen2-instruct model - may take a minute the first time.") + +qwen2 = ModelCatalog().load_model("qwen2-1.5b-instruct-gguf", max_output=200) +response = qwen2.inference("I am going to visit Istanbul. What should I see?") +print("\nresponse: ", response) + +# 2 - RAG FINETUNE - DRAGON + BLING + +print("\nExample #2 - RAG finetuned Qwen2 for fact-based question answering with context passage.") + +qwen2_rag_finetunes = ["dragon-qwen-7b-gguf", "bling-qwen-1.5b-gguf", "bling-qwen-0.5b-gguf"] + +qwen2_rag = ModelCatalog().load_model("bling-qwen-1.5b-gguf", temperature=0.0, sample=False) +context = "The stock is now soaring to $120 per share after great earnings." +response = qwen2_rag.inference("What is the current stock price?", add_context=context) + +print("\nqwen2-rag response: ", response) + + +# 3 - FUNCTION-CALLING EXTRACTION SLIM MODELS + +print("\nExample #3 - Qwen2 Extract function calling model.") + +qwen2_extract_function_calls = ["slim-extract-qwen-1.5b-gguf", "slim-extract-qwen-0.5b-gguf"] + +context_passage = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + +qwen2_extract = ModelCatalog().load_model("slim-extract-qwen-1.5b-gguf",temperature=0.0,sample=False) +response = qwen2_extract.function_call(context_passage, params=["earnings per share"]) + +print("\nqwen2-extract response: ", response) + diff --git a/solutions/models/using_function_calls.py b/solutions/models/using_function_calls.py new file mode 100644 index 0000000..1b8212d --- /dev/null +++ b/solutions/models/using_function_calls.py @@ -0,0 +1,200 @@ + +""" Function Calling with SLIMs - this example illustrates how to move beyond basic question-answer prompting, +and begin to integrate function calls into LLM-based workflows. + + Generally, function-calling is a specialized capability of frontier language models, such as OpenAI GPT4. + + We have adapted this concept to small language models through SLIMs (Structured Language Instruction Models), + which are 'single function' models fine-tuned to accept three main inputs to construct a prompt: + + As of June 2024, there are 18 distinct SLIM function calling models with many more on the way, for most common + extraction, classification, and summarization tasks. + + All SLIM models have a common prompting structure + + Inputs: + -- text passage - this is the core passage or piece of text that you would like the model to assess + -- function - classify, extract, generate - this is handled by default by the model class, so usually does + not need to be explicitly declared - but is an option for SLIMs that support more than one function + -- params - depends upon the model, used to configure/guide the behavior of the function call - optional for + some SLIMs + + Outputs: + -- structured python output, generally either a dictionary or list + + Main objectives: + -- enable function calling with small, locally-running models, + -- simplify prompts by defining specific functions and fine-tuning the model to respond accordingly + without 'prompt magic' + -- standardized outputs that can be handled programmatically as part of a multi-step workflow. + """ + + +from llmware.models import ModelCatalog + + +def discover_slim_models(): + + """ Discover a list of SLIM tools in the Model Catalog. + + -- SLIMs are available in both traditional Pytorch and quantized GGUF packages. + -- Generally, we train/fine-tune in Pytorch and then package in 4-bit quantized GGUF for inference. + -- By default, we designate the GGUF versions with 'tool' or 'gguf' in their names. + -- GGUF versions are generally faster to load, faster for inference and use less memory in most environments.""" + + tools = ModelCatalog().list_llm_tools() + tool_map = ModelCatalog().get_llm_fx_mapping() + + print("\nList of SLIM model tools (GGUF) in the ModelCatalog\n") + + for i, tool in enumerate(tools): + model_card = ModelCatalog().lookup_model_card(tool_map[tool]) + print(f"{i} - tool: {tool} - " + f"model_name: {model_card['model_name']} - " + f"model_family: {model_card['model_family']}") + + return 0 + + +def hello_world_slim(): + + """ SLIM models can be identified in the ModelCatalog like any llmware model. Instead of using + inference method, SLIM models are used with the function_call method that prepares a special prompt + instruction, and takes optional parameters. + + This example shows a series of function calls with different SLIM models. + + Please note that the first time the models will be pulled from the llmware Huggingface repository, and will + take a couple of minutes. Future calls will be much faster once cached in memory locally. """ + + print("\nExecuting Function Call Inferences with SLIMs\n") + + # Sentiment Analysis + + passage1 = ("This is one of the best quarters we can remember for the industrial sector " + "with significant growth across the board in new order volume, as well as price " + "increases in excess of inflation. We continue to see very strong demand, especially " + "in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would " + "be accumulating more stock on any dips.") + + # here are the two key lines of code + model = ModelCatalog().load_model("slim-sentiment-tool") + response = model.function_call(passage1) + + print("sentiment response: ", response['llm_response']) + + # Named Entity Recognition + + passage2 = "Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s." + + model = ModelCatalog().load_model("slim-ner-tool") + response = model.function_call(passage2) + + print("ner response: ", response['llm_response']) + + # Extract anything with Slim-extract + + passage3 = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + + model = ModelCatalog().load_model("slim-extract-tool") + response = model.function_call(passage3, function="extract", params=["revenue growth"]) + + print("extract response: ", response['llm_response']) + + # Generate questions with Slim-Q-Gen + + model = ModelCatalog().load_model("slim-q-gen-tiny-tool", temperature=0.2, sample=True) + # supported params - "question", "multiple choice", "boolean" + response = model.function_call(passage3, params=['multiple choice']) + + print("question generation response: ", response['llm_response']) + + # Generate topic + + model = ModelCatalog().load_model("slim-topics-tool") + response = model.function_call(passage3) + + print("topics response: ", response['llm_response']) + + # Generate headline summary with slim-xsum + model = ModelCatalog().load_model("slim-xsum-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("xsum response: ", response['llm_response']) + + # Generate boolean with optional '(explain)` in parameter + model = ModelCatalog().load_model("slim-boolean-tool") + response = model.function_call(passage3, params=["Did Adobe revenue increase? (explain)"]) + + print("boolean response: ", response['llm_response']) + + # Generate tags + model = ModelCatalog().load_model("slim-tags-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("tags response: ", response['llm_response']) + + return 0 + + +def using_logits_and_integrating_into_process(): + + """ This example shows two key elements of function calling SLIM models - + + 1. Using Logit Information to indicate confidence levels, especially for classifications. + 2. Using the structured dictionary generated for programmatic handling in a larger process. + + """ + + print("\nExample: using logits and integrating into process\n") + + text_passage = ("On balance, this was an average result, with earnings in line with expectations and " + "no big surprises to either the positive or the negative.") + + # two key lines (load_model + execute function_call) + additional logit_analysis step + sentiment_model = ModelCatalog().load_model("slim-sentiment-tool", get_logits=True) + response = sentiment_model.function_call(text_passage) + analysis = ModelCatalog().logit_analysis(response,sentiment_model.model_card, sentiment_model.hf_tokenizer_name) + + print("sentiment response: ", response['llm_response']) + + print("\nAnalyzing response") + for keys, values in analysis.items(): + print(f"{keys} - {values}") + + # two key attributes of the sentiment output dictionary + sentiment_value = response["llm_response"]["sentiment"] + confidence_level = analysis["confidence_score"] + + # use the sentiment classification as a 'if...then' decision point in a process + if "positive" in sentiment_value: + print("sentiment is positive .... will take 'positive' analysis path ...", sentiment_value) + else: + print("sentiment is negative .... will take 'negative' analysis path ...", sentiment_value) + + if "positive" in sentiment_value and confidence_level > 0.8: + print("sentiment is positive with high confidence ... ", sentiment_value, confidence_level) + + return 0 + + +if __name__ == "__main__": + + # discovering slim models in the llmware catalog + discover_slim_models() + + # running function call inferences + hello_world_slim() + + # doing interesting stuff with the output + using_logits_and_integrating_into_process() + + diff --git a/solutions/models/using_local_foundry_models.py b/solutions/models/using_local_foundry_models.py new file mode 100644 index 0000000..44b19a3 --- /dev/null +++ b/solutions/models/using_local_foundry_models.py @@ -0,0 +1,38 @@ + +""" This example shows how to use Windows Local Foundry models. + + pre-reqs: + + 1. install local foundry, e.g., `winget install Microsoft.FoundryLocal` + 2. pip3 install foundry-local-sdk + 3. pip3 install openai (openai api used, but not openai model - all runs locally) +""" + +from llmware.models import ModelCatalog, WindowsLocalFoundryHandler + +# activate the connection and poll foundry-local instance for available models +foundry_handler = WindowsLocalFoundryHandler() +cat = foundry_handler.activate_catalog(True) + +# foundry local models now added to the catalog +foundry_models = ModelCatalog().list_models_by_type("WindowsLocalFoundryModel") +for i, mod in enumerate(foundry_models): + print("--foundry model - ", mod) + +all_models = ModelCatalog().list_all_models() + +# load foundry model like any other in llmware +# note: this example was used on a Windows Intel x86 Lunar Lake +# -- different platforms will have different supported models + +m1 = "Phi-3.5-mini-instruct-openvino-gpu:1-foundry" +m2 = "qwen2.5-0.5b-instruct-generic-cpu:4-foundry" +model = ModelCatalog().load_model(m1, max_output=500) + +for token in model.stream("What are the best sites to see in Rome?"): + print(token, end="") + +# stop foundry local server when done +WindowsLocalFoundryHandler().stop_server() + + diff --git a/solutions/models/using_slim_extract.py b/solutions/models/using_slim_extract.py new file mode 100644 index 0000000..9287845 --- /dev/null +++ b/solutions/models/using_slim_extract.py @@ -0,0 +1,417 @@ + +""" This example illustrates how to use the slim-extract model to extract custom keys from selected text. + We have included a set of sample earnings releases (comprising lines ~10 - ~385 of this script), and run a + simple loop through the earnings releases, showing how to create an extract prompt to identify + 'revenue growth' from these examples. + + There are several function-calling models in the slim-extract family, fine-tuned on multiple leading + small model base foundations - full list and options are below in the code. """ + +from llmware.models import ModelCatalog + +# Sample earnings releases + +earnings_releases = [ + + {"context": "Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee."}, + + {"context": "Dick’s Sporting Goods raised its dividend by 10% on Thursday as the company posted its largest sales " + "quarter in its history and projected another year of growth. The company’s shares jumped more than " + "15% in intraday trading. CEO Lauren Hobart said on an earnings call Thursday that Dick’s sales " + "growth came from bigger tickets — either higher prices or more expensive items — as its transactions " + "were flat. Many retailers benefited from a 53rd week in fiscal 2023, but Dick’s said it still broke " + "records during its fiscal fourth quarter even without those extra days. Here’s how the athletic " + "apparel retailer did compared with what Wall Street was anticipating, based on a survey of " + "analysts by LSEG, formerly known as Refinitiv: Earnings per share: $3.85 adjusted vs. $3.35 expected " + "Revenue: $3.88 billion vs. $3.80 billion expected The company’s reported net income for the three-month " + "period that ended Feb. 3 was $296 million, or $3.57 per share, compared with $236 million, or $2.60 a " + "share, a year earlier. Excluding one-time items related to impairment charges and inventory write-offs, " + "Dick’s reported earnings per share of $3.85. Sales rose to $3.88 billion, up about 8% from $3.60 billion " + "a year earlier. “With our industry-leading assortment and strong execution, we capped off the year " + "with an incredibly strong fourth quarter and holiday season,” Hobart said in a statement. “We are " + "guiding to another strong year in 2024. We plan to grow both our sales and earnings through " + "positive comps, higher merchandise margin and productivity gains,” she added. During the quarter, " + "same-store sales rose 2.8%, well ahead of the 0.8% lift that analysts had expected, according to " + "StreetAccount. “Growth in transactions” and market share gains drove the increase, said Executive " + "Chairman Ed Stack."}, + + {"context": "Comcast topped both revenue and profit estimates in the fourth quarter as it lost fewer broadband " + "subscribers than expected, and it raised its dividend 7%, the company said Thursday. " + "Here’s how Comcast performed, compared with estimates from analysts surveyed by LSEG, " + "formerly known as Refinitiv. Earnings per share: 84 cents adjusted vs. 79 cents expected " + "Revenue: $31.25 billion vs. $30.51 billion expected For the quarter ended Dec. 31, net " + "income rose 7.8% to $3.26 billion, or 81 cents a share, compared with $3.02 billion, or " + "70 cents a share, a year earlier. Revenue increased 2.3% compared with the prior-year period. " + "Adjusted earnings before interest, taxes, depreciation and amortization (EBITDA) was flat year " + "over year at about $8 billion. 'For the third consecutive year, we generated the highest revenue, " + "adjusted EBITDA and adjusted EPS in our company’s history', Comcast Chief Executive Officer Brian " + "Roberts said in a statement. 'We also reported the highest adjusted EBITDA on record at Theme Parks; " + "were the #1 studio in worldwide box office for the first time since 2015; and maintained Peacock’s " + "position as the fastest growing streamer in the U.S.'"}, + + {"context": "Dollar General forecast annual sales above Wall Street estimates on Thursday, banking on higher " + "demand from inflation-hit customers buying groceries and essentials from the discount retailer’s stores. " + "Shares of the company rose about 6% in early trading, after falling nearly 45% in 2023 on rising costs " + "and stiff competition from bigger retailers. But higher prices and borrowing costs have prompted " + "budget-conscious consumers to cook more meals at home, helping Dollar General record stronger " + "footfall at its outlets as shoppers hunt for lower-margin, needs-based goods, over pricier general " + "merchandise. “With customer traffic growth and market share gains during the quarter, we believe our " + "actions are resonating with customers,” CEO Todd Vasos said in a statement. Vasos’s strategy - to focus " + "on the basics, like more employee presence at stores, greater customer engagement and expanding " + "private-label brands - has helped stabilize Dollar General’s business. Over the last few quarters, " + "Dollar General and rival Dollar Tree have struggled with rising costs linked to their supply " + "chains, labor and raw materials, while facing tough competition from retailers like Walmart " + "and Chinese ecommerce platform Temu. Dollar Tree’s shares fell more than 15% on Wednesday, after it " + "forecast weak sales and profit for 2024 and laid out plans to shutter 970 of its Family Dollar " + "stores. “Dollar General has a much rosier outlook than Dollar Tree... Dollar Tree’s challenges " + "with Family Dollar were years in the making, while Dollar General has embarked on an aggressive " + "effort to add more frozen, refrigerated and fresh produce,” eMarketer senior analyst Zak Stambor said. " + "Dollar General forecast 2024 sales to grow between 6.0% and 6.7%, above analysts’ estimate of 4.4% " + "growth to $40.33 billion, according to LSEG data. It still sees annual per-share profit between " + "$6.80 and $7.55, compared with estimates of $7.55. Its fourth-quarter net sales of $9.86 billion " + "surpassed estimates of $9.78 billion. It also reported an estimate-beating profit of $1.83 per share."}, + + {"context": "Shares of Zara owner Inditex hit record highs on Wednesday according to LSEG data, climbing over 6% during " + "intraday trading after the company announced its 2023 full-year results. As of 11:50 London time, shared " + "were just over 6% higher at 43.58 euros, or $47.69. Sales increased 10.4% to 35.9 billion euros for the " + "year, the company said, signaling this was a record high. Sales grew across all geographic regions and " + "across Inditex’s brands and were “very satisfactory,” both online and in store, the company said. A total of " + "5,692 stores were operational at the end of the year, Inditex said, adding it plans to expand further in " + "2024, including with Zara shops in Los Angeles and Las Vegas. The company also plans to open new distribution " + "centers in 2024 and 2025, as part of a major logistics expansion plan that will cost the company " + "investments of 900 million euros in both years. Net income also reached a fresh high after soaring " + "30.3% from 2022 to reach 5.4 billion euros last year. The company’s gross profit came in at 20.8 billion " + "euros, up 11.9% on the year. “Inditex’s performance in 2023 has been excellent. Our teams have been able to " + "take advantage of the opportunities to keep growing profitably. We are investing to drive future growth and " + "continue to offer an attractive remuneration to shareholders,” Inditex CEO Oscar García Maceiras said in a " + "statement. The Spanish clothing company owns a range of vastly popular brands including household name " + "Zara, as well as Pull & Bear, Bershka, Stradivarius, premium retailer Massimo Dutti and sports and the " + "athleisure-focused Oysho. Zara, including the Zara Home range, was the biggest contributor to sales in " + "2023, followed by Pull & Bear and Massimo Dutti, Inditex said Wednesday. The company also indicated " + "that 2024 was off to a strong start, with sales in constant currency up 11% over the Feb. 1 to March 11 " + "stretch, compared with the same period a year earlier."}, + + {"context": "Oracle reported quarterly earnings on Monday that exceeded Wall Street’s expectations. Shares rose " + "13% in extended trading. Here’s how the company did in the fiscal third quarter ending Feb. 29, compared " + "to estimates by LSEG, formerly known as Refinitiv: Earnings per share: $1.41 adjusted vs. $1.38 expected " + "Revenue: $13.28 billion vs. $13.3 billion expected For the fiscal fourth quarter, Oracle said it expects " + "earnings of $1.62 to $1.66 per share. Analysts were expecting $1.64 in adjusted earnings per share, according " + "to LSEG. Revenue growth will be between 4% and 6% over sales of $13.8 billion a year ago. The midpoint of that " + "range would equal revenue of about $14.5 billion, while analysts were expecting a little more than $14.7 billion. " + "Oracle CEO Safra Catz said the company was committed to hitting previously stated goals of $65 billion in " + "sales by fiscal 2026. “Some of these goals might prove to be too conservative given our momentum,” Catz said. " + "Revenue rose 7% in the quarter from $12.4 billion a year earlier. Net income climbed 27% to $2.4 billion, " + "or 85 cents per share, from $1.9 billion, or 68 cents per share, a year ago. Oracle’s cloud services and " + "license support segment, its largest business, saw sales rise 12% to $9.96 billion, slightly beating " + "StreetAccount consensus expectations of $9.94 billion. The company attributed the rise to strong demand " + "for its artificial intelligence servers. Catz said the company added several “large new cloud " + "infrastructure” contracts during the quarter. The company’s cloud revenue, which is reported as part " + "of the cloud services unit, rose 25% year over year to $5.1 billion, Oracle said. “We signed several large " + "deals this quarter and we have many more in the pipeline,” Catz told investors on the earnings call. " + "Oracle Chairman Larry Ellison cited increased business from Microsoft on the earnings call. " + "“We’re building 20 data centers from Microsoft and Azure. They just ordered three more data centers " + "this week,” Ellison said. The company’s other units didn’t fare as well. Cloud license and on-premise sales " + "declined 3% to $1.26 billion, slightly beating StreetAccount’s forecast. Hardware revenue fell 7% to " + "$754 million, while sales in the company’s services division slid 5% to $1.31 billion, both falling short " + "of StreetAccount expectations. Prior to Monday’s report, Oracle shares were up 8.7% for the year, " + "slightly outperforming the S&P 500."}, + + {"context": "Porsche on Tuesday warned that profitability will decline this year as it launches new models amid " + "tough economic conditions, but hiked its dividend on the back of a rise in 2023 operating profit. The German " + "luxury automaker said it expects an operating return on sales of between 15% and 17% in 2024, down from the " + "18% margin notched in 2023 and 2022. In the long term, the group targets an operating return on sales of more " + "than 20%. Explaining the more cautious profitability outlook, the company cited “the comprehensive renewal " + "of its product range in 2024, the global framework conditions, higher depreciations on capitalized " + "development costs and the continued investments in the brand and the Porsche ecosystem.” The company’s shares " + "were around 4.8% higher by early afternoon, having reversed opening losses of more than 2%. Porsche is " + "launching four new car ranges in 2024 in the form of the Panamera, Macan, Taycan and 911 model lines. " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China WATCH NOW VIDEO 03:17 " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China “2024 is going to be a year of " + "product launches for Porsche – more so than any year in our history,” Chairman Oliver Blume said in a " + "statement. “We will be introducing a variety of exhilarating sports cars to the road, they will delight " + "our customers around the world. This will put the wind at our back for years to come.”"}, + + {"context": "Lego on Tuesday reported its full-year 2023 results, saying it’s revenue grew by 2% throughout the year, " + "in line with expectations. The company made “very, very strong progress” and “grew comfortably” in the " + "U.S., its CEO Niels Christiansen told CNBC. The toy industry has been struggling to maintain " + "pandemic-era growth as inflation is putting pressure on demand and sales. In-store participation is greater " + "than prior to the pandemic, Lego CEO says In-store participation is greater than prior " + "to the pandemic, Lego CEO says The chief executive of Denmark’s Lego on Tuesday reflected on a tough year " + "for the world’s largest toymaker, and outlined the firm’s long-term plans to stay relevant and “cool with kids. ”" + "Lego said its 2023 revenue was 2% higher compared to the previous year, growing to 65.9 billion Danish krone " + "(around $9.65 billion). This was in line with expectations, Lego said in a statement. “It was a difficult year,” " + "Lego CEO Niels Christiansen told CNBC. However, he said the company had “managed to take quite a bit of " + "market share.” The Danish toymaker said operating profit declined slightly from 17.9 billion Danish krone " + "to 17.1 billion, noting that it had boosted spending on strategic initiatives designed to drive growth. " + "Net profit came in at 13.1 billion Danish krone in 2023, compared to 13.8 billion the previous year. " + "Consumer sales were up 4% despite slumping in China, Lego said, attributing the growth to increasing demand " + "in the U.S. and central and eastern Europe. It comes as the wider toy industry has been struggling to " + "maintain growth after booming during the coronavirus pandemic, when parents looked for new ways to " + "entertain their children and adults re-discovered childhood pastimes. Toy company Hasbro earlier this month " + "said its 2023 revenue fell by 15% compared to 2022 and that it expected to see a further decline this year."}, + + {"context": "Adidas on Wednesday warned of a sales decline in its overstocked North American market in 2024, as the " + "German sportswear brand continues to sell off its remaining Yeezy inventory. Currency-neutral sales in " + "North America are expected to decline to a mid-single-digit rate in 2024, but are projected to notch " + "mid-single-digit growth worldwide despite persistent “macroeconomic challenges and geopolitical tensions,” " + "the company said. Adidas confirmed its 2023 operating profit came in at 268 million euros ($292.9 million) " + "on the back of flat currency-neutral sales, significantly above prior expectations as the company continues " + "to take a hit from the cessation of its line of Yeezy — footwear the retailer produced in a collaboration with " + "American rapper Ye, formerly known as Kanye West. For the fourth quarter, the company posted an operating " + "loss of 377 million euros. The board proposed a flat dividend of 0.70 euros per share. “Although by far not " + "good enough, 2023 ended better than what I had expected at the beginning of the year,” CEO Bjørn Gulden " + "said in a statement. “Despite losing a lot of Yeezy revenue and a very conservative sell-in strategy, " + "we managed to have flat revenues. We expected to have a substantial negative operating result, but " + "achieved an operating profit of €268 million.” Adidas was confirming preliminary results released in late " + "January, when it announced that it would not write off the majority of its Yeezy inventory and would instead " + "sell off the remaining shoes at cost. The sportswear giant was forced to axe the Yeezy line after terminating " + "its partnership with Ye over a string of anti-Semitic remarks that the rapper made in 2022. Adidas said the " + "discontinuation of Yeezy represented a drag of around 500 million euros in the year-on-year comparison " + "through 2023, though the sale of parts of the remaining inventory in the second and third quarter positively " + "impacted net sales by around 750 million euros. “With a very disciplined go-to-market and buying process, " + "we reduced our inventories by almost €1.5 billion. With the exception of the U.S., we now have healthy " + "inventories everywhere,” Gulden said. He added that the company is expecting some growth in the " + "first quarter of 2024 and a further pick-up in the second half of the year. “We still have a lot of work " + "to do, but I feel very confident we are on the right track. We will bring adidas back again. Give us some " + "time and we will again say – we got this!” he said. Adidas projected an operating profit of around " + "500 million euros in 2024, with unfavorable currency effects expected to “weigh significantly on the " + "company’s profitability” because of adverse impacts on both reported revenues and gross margin development." + "Adidas shares were flat by mid-morning on Wednesday. Mamta Valechha, equity research analyst at " + "Quilter Cheviot, said that, given that the headline numbers were already pre-released in January, the most " + "interesting aspect of Wednesday’s report was the “clear acceleration of the Adidas brand.”"}, + + {"context": "Costco on Thursday missed Wall Street’s revenue expectations for its holiday quarter, despite reporting " + "year-over-year sales growth and strong e-commerce gains. Shares of the retailer fell about 4% in aftermarket " + "trading. The company’s stock had hit a 52-week high earlier in the day. Here’s what Costco reported for its " + "fiscal second quarter of 2024 compared with what Wall Street was expecting, based on a survey of analysts by " + "LSEG, formerly known as Refinitiv: Earnings per share: $3.92 vs. $3.62 expected Revenue: $58.44 billion vs. " + "$59.16 billion expected In the three-month period that ended Feb. 18, Costco’s net income rose to " + "$1.74 billion, or $3.92 per share, compared with $1.47 billion, or $3.30 per share, a year earlier. " + "Costco’s revenue for the quarter increased from $55.27 billion in the year-ago period. Comparable sales for " + "the company increased 5.6% year over year and 4.3% in the U.S. Excluding changes in gas prices and foreign " + "currency, the metric increased 5.8% overall and 4.8% in the U.S. Sales of food and sundries, a category " + "that includes snack foods and beverages, were up by mid single digits in the quarter, CFO Richard Galanti " + "said on the company’s earnings call. Fresh foods were up high single digits and nonfoods were up mid single " + "digits. Ancillary businesses, which includes more service-related purchases like travel, were up by low " + "single digits, he said. Costco’s food court, pharmacy and optical centers were top performers in the quarter " + "and gas was down low single digits as the price per gallon fell. More shoppers came to Costco, and they " + "spent more on their shopping trips during the quarter. Traffic increased 5.3% across the globe and 4.3% in " + "the U.S., Galanti said on the earnings call. The average ticket increased in the U.S. and worldwide, he " + "said. Inflation was roughly flat year over year in the quarter, which allowed the retailer to reduce " + "prices for some items, Galanti said. For example, he said, it’s been able to cut the price of reading " + "glasses from $18.99 to $16.99 and slash the price of a 48 count of Kirkland Signature batteries from " + "$17.99 to $15.99. In the prior quarter, he said inflation was as much as 1% year over year. Galanti said many " + "new items in categories like sporting goods and lawn and garden will also have lower prices compared with " + "a year ago because of falling freight and commodity costs. Costco has 875 warehouses, including 603 in " + "the U.S. and Puerto Rico. It also has clubs in about a dozen other countries, including Canada, Mexico, " + "Japan and China. In the second quarter, Costco opened four new clubs, including three in the U.S. " + "and one in Shenzhen, China. That marked its sixth club to open in China, Galanti said. Two of the three " + "new U.S. locations were Costco Business Centers, which are specifically geared toward small business " + "owners like restaurant operators. As of Thursday’s close, Costco shares have risen nearly 19% since the " + "start of the year. The stock touched a 52-week high of $787.08 earlier in the day and closed at $785.59, " + "bringing the company’s market value to nearly $350 billion."}, + + {"context": "Shares of Teleperformance plunged 23% on Thursday, after the French call center and office services group " + "missed its full-year revenue target and flagged a “volatile economic environment.” Investors have been " + "spooked by the potential impact of artificial intelligence on its business model, as companies become more " + "able to tap into the technology directly for their own benefit. Teleperformance shares dropped 16% last " + "week, according to LSEG data, after Swedish financial services company Klarna said its Open AI-powered " + "customer service assistant was handling two-thirds of customer service calls. But Teleperformance CEO " + "Daniel Julien on Thursday said that AI would be a positive for its business model — and that it will never " + "fully replace the value of human interaction. “AI is part of the solutions we provide to the clients,” " + "Julien told CNBC’s “Squawk Box Europe.” “AI helps to increase the accuracy of our employees ... which is " + "great, but at the end of the day we are here to reduce the friction between the citizens, or the customer, " + "and the companies they have bought a product and service from.” He stressed, “It’s not just a transactional " + "relationship, it has a lot to do with reassuring, with trust, with empathy. So we perceive AI as enhancing " + "the job that our human employees do, but absolutely not replacing them.” hide content Teleperformance SE " + "RT Quote | Exchange | EUR 87.16 quote price arrow up+0.68 (+0.79%) Last | 03/15/24 CET WATCHLIST + QUOTE DETAILS " + "Teleperformance share price. Teleperformance reported 2.3% higher revenue at 8.345 billion euros " + "($9.091 billion) in 2023, as net profit fell year-on-year from 643 million euros to 602 million euros. " + "Diluted earnings per share hit 10.18 euros, down from 10.77 euros. In its results, the company said it is " + "working with clients on 250 AI projects, including in generative AI, and it has expanded its portfolio " + "with new partnerships in the space. “Even the most high-tech or the most AI-involved companies are clients " + "of Teleperformance. We chose that there is a complementarity and not separation,” Julien told CNBC, " + "flagging the company’s agreement with tech giant and major AI player Microsoft. “They are there to provide " + "a solution that is going to augment the productivity, augment the quality of the information that can be " + "given to the customer, but, at the end of the day, the customer is a human being. The day the customer is " + "going to be a robot, maybe AI will replace the humans.”"}, + + {"context": "CrowdStrike shares surged as much as 21% in after-hours trading Tuesday after the cybersecurity " + "company reported a beat on the top and bottom lines, plus issued stronger-than-expected guidance for " + "the upcoming quarter and full year. Here’s how the company did compared to consensus estimates " + "based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per share: 95 cents " + "adjusted vs. 82 cents expected Revenue: $845 million vs. $839 million expected For the period that " + "ended Jan. 31, CrowdStrike saw net income of $54 million, or 22 cents per share, from a " + "$48 million loss, or a 20 cent loss per share, in the year-ago period. CrowdStrike has now " + "reported GAAP net income for the past four quarters, Chief Financial Officer Burt Podbere " + "said in the earnings release. Full-year revenue rose 36% year over year, from $2.24 billion " + "to $3 billion. The company also announced it would acquire Flow Security for an undisclosed " + "price in a cash-and-stock deal, slated to close in the company’s fiscal first quarter. " + "The company has been stepping up its merger and acquisition activity in recent months. “CrowdStrike " + "is cybersecurity’s consolidator of choice, innovator of choice, and platform of choice to " + "stop breaches,” co-founder and CEO George Kurtz said in a release. The company also guided to " + "fiscal first-quarter revenue between $902 million and $906 million, better than a consensus " + "estimate of $899 million. CrowdStrike also expects earnings per share for the period between " + "89 cents and 90 cents, better than the consensus estimate of 82 cents. Podbere also reiterated " + "the company’s focus on achieving $10 billion in annual recurring revenue by 2030. The company " + "reached $3.4 billion in annual recurring revenue in January."}, + + {"context": "Shares of Amer Sports, the maker of Wilson tennis rackets and Lousiville Slugger baseball " + "bats, fell on Tuesday after the company reported strong sales in China but a slowdown in " + "wholesale orders. Here’s how the newly public athletic company did in its fourth quarter. " + "CNBC didn’t compare the results to Wall Street estimates because it’s the first earnings " + "report since Amer Sports went public. Loss per share: 25 cents Revenue: $1.32 billion In the " + "three months ended Dec. 31, the company reported a net loss of $94.9 million, or 25 cents " + "per share, compared with $148.3 million, or 39 cents per share, a year earlier. Sales rose to " + "$1.32 billion, up about 10% from $1.2 billion a year earlier. Shares closed about 5% lower. " + "Amer, which also owns Arc’teryx, Salomon, and a number of other athletic equipment and " + "apparel brands, operates in three distinct business segments. They are technical apparel, " + "which includes its pricey Arc’teryx winter jackets; outdoor performance, such as Salomon’s " + "winter sports equipment; and ball and racquet sports, which includes equipment and apparel " + "from Wilson and Louisville, among others. During the quarter, sales for Amer’s technical " + "apparel rose 26% year over year to $550 million, driven by a 42% jump in direct sales. " + "Sales in the segment primarily come from shoppers who are buying directly from Amer’s " + "brands rather than from wholesale partners. Sales for outdoor performance increased 2% to " + "$523 million, driven by strength in the segment’s winter sports equipment franchise, " + "which was offset by a slowdown in wholesale orders for Salomon footwear. Ball and racquet sales " + "declined 3% to $242 million as the segment lapped tougher comps. In the year-ago period, " + "retailers were still dealing with supply chain issues and had over-ordered equipment like " + "tennis rackets and baseball bats. As they looked to keep their inventory levels in check, " + "some wholesalers pulled back on orders during the quarter, but Amer expects the segment " + "will level out in the quarters ahead and end fiscal 2024 with sales up in the low- to " + "mid-single digit range. The company started trading on the New York Stock Exchange last " + "month under the ticker “AS.” The shares rose just 3% in Amer’s debut on the public " + "markets after it priced its IPO at a discount. Sellers showed muted interest in the " + "stock during its first day of trading over concerns about its connections and exposure to " + "China and its debt-laden balance sheet. Founded in Helsinki in 1950, Amer was a Finnish public " + "company until it was taken private in 2019 by a consortium of investors led by China’s Anta " + "Sports, FountainVest Partners, Anamered Investments and Tencent. Since the acquisition, " + "sales grew about 45% from $2.45 billion in 2020 to $3.55 billion in 2022. Revenue jumped " + "again in 2023 to $4.37 billion, the company said Tuesday."}, + + {"context": "Shares of Dell Technologies popped more than 15% during extended trading Thursday after the company " + "released fourth-quarter results that beat analysts’ estimates and showed strong demand for its " + "artificial intelligence servers. Here’s how the company did: Earnings per share: $2.20 adjusted vs. " + "$1.73 expected by LSEG, formerly known as Refinitiv Revenue: $22.32 billion vs. $22.16 billion " + "expected by LSEG Dell’s revenue for the fiscal 2024 fourth quarter fell 11% from $25.04 billion " + "in the year-ago quarter. The company reported net income $1.16 billion, up 89% from the $614 million " + "it posted in the same period last year. Chief Financial Officer Yvonne McGill said in a release " + "that the company is increasing its annual dividend by 20% to $1.78 per share, which she called " + "a “testament to our confidence in the business.” Dell’s Infrastructure Solutions Group (ISG) " + "reported $9.3 billion in revenue for the quarter, down 6% year over year but up 10% from the " + "third quarter. Servers and networking revenue made up the bulk of that, with $4.9 billion in " + "revenue driven by “AI-optimized servers.” Storage revenue came in at $4.5 billion. The company’s " + "Client Solutions Group (CSG) reported $11.7 billion for the quarter, down 12% year over year. " + "That includes $9.6 billion in commercial client revenue, which fell 11% since the fourth quarter " + "of last year, and $2.2 billion in consumer revenue, down 19% year over year. “Our strong AI-optimized " + "server momentum continues, with orders increasing nearly 40% sequentially and backlog nearly " + "doubling, exiting our fiscal year at $2.9 billion,” Chief Operating Officer Jeff Clarke " + "said in the release. For its first quarter, Dell said during its quarterly call with " + "investors that it expects to report revenue between $21 billion and $22 billion. The company " + "said it is encouraged by momentum around AI, and that it expects to return to growth for " + "fiscal 2025. However, the company noted that the macroeconomic environment is causing some " + "customers to be cautious about infrastructure costs."}, + + {"context": "Birkenstock on Thursday beat holiday quarter revenue expectations, reporting a 22% year-on-year " + "jump, as the German sandal company benefited from higher pricing and rising U.S. demand. As a newly " + "public company, Birkenstock is still getting into a public reporting rhythm and only just " + "released its fiscal 2023 results and 2024 guidance a little over a month ago. On Thursday, " + "it said it stands by guidance issued then and still expects sales to be between 1.74 billion " + "euros and 1.76 billion euros ($1.89 billion and $1.91 billion), representing growth of 17% to 18%. " + "The shoemaker, which started trading on the New York Stock Exchange under the ticker “BIRK” in " + "October, saw a muted debut when it first hit the public markets, with shares sliding more than " + "12% on its first day as a public company. The stock has since rebounded and is up more than 5% " + "this year, as of the Wednesday close. Birkenstock’s shares closed more than 2% lower Thursday. " + "Here’s how the shoemaker did in its fiscal first quarter compared with what Wall Street was " + "anticipating, based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per " + "share: 9 euro cents adjusted vs. 9 euro cents expected. Revenue: 302.9 million euros vs. 288.7 " + "million euros expected. The company reported a net loss of 7.15 million euros for the " + "three-month period that ended Dec. 31, or a loss of 4 euro cents per share. A year earlier, " + "it reported a loss of 9.19 million euros, or a loss of 5 euro cents per share. " + "Excluding one-time items, Birkenstock reported a profit of 17 million euros, or " + "9 euro cents per share. Sales rose to 302.9 million euros, up 22% from 248.5 million euros " + "a year earlier. Adjusted earnings before interest, taxation, depreciation and amortization " + "(EBITDA) rose 12% year on year to 81 million euros, with an adjusted EBITDA margin of 26.9%, " + "down from 29.1% a year earlier. The retailer has been making strides to grow its direct-to-consumer " + "business, which comes with better profits and more customer insights than relying on wholesale partners. " + "CEO Oliver Reichert has said the company deliberately engineers its distribution strategy so " + "demand is higher than supply but it’s working to double its production capabilities over " + "the next three years to narrow that gap. The chief executive said those investments, " + "along with other efforts the company is undertaking to drive growth, is having a “planned” " + "but “temporary” impact to profitability. The company’s gross profit margin inched down to " + "61% from 61.7% during the same period last year, with Birkenstock citing “unfavorable " + "currency translation and the planned, temporary under-absorption from our ongoing " + "capacity expansion.” The company said it continues to carefully track input costs and " + "is mitigating inflationary pressures with “executed, selective price increases.” In Europe, the " + "company said it had “two consecutive price adjustments” with “no signs of rejection.”"}, + + {"context": "Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter."} + +] + +# *** Execution script starts here *** + +# specialized function-calling extraction models on top of several leading small model bases, +# ranging from 0.5b (qwen2) - 3.8b (phi3) + +slim_extract_models = ["slim-extract-tool", # original - stablelm-3b (2.7b) + "slim-extract-tiny-tool", # tiny-llama 1.1b + "slim-extract-qwen-1.5b-gguf", # **NEW** qwen 1.5b + "slim-extract-phi-3-gguf", # **NEW** phi-3 (3.8b) + "slim-extract-qwen-0.5b-gguf"] # **NEW** qwen 0.5b + +# load the model +model = ModelCatalog().load_model("slim-extract-tool",sample=False,temperature=0.0, max_output=100) + +# iterate through the earnings release samples above +for i, sample in enumerate(earnings_releases): + + # key line: execute function_call on selected model with 'custom_key' = "revenue growth" + response = model.function_call(sample["context"], function="extract", params=["revenue growth"]) + + # display the response on the screen + print("extract response: ", i, response["llm_response"]) + diff --git a/solutions/onnxruntime/adding_openvino_or_onnx_model.py b/solutions/onnxruntime/adding_openvino_or_onnx_model.py new file mode 100644 index 0000000..659f9c3 --- /dev/null +++ b/solutions/onnxruntime/adding_openvino_or_onnx_model.py @@ -0,0 +1,144 @@ + +""" This example shows how to add a custom or private OpenVino or ONNX model to the llmware model catalog. + + Over the next few releases, we will be expanding the default ModelCatalog considerably, but for the time + being, please feel free to follow the steps below to build your own custom catalog. + + We show below templates for the model card dictionaries - most of which is fairly easy to build for a given + model. + + We highlight both the main step - which is a simple one-liner to register the model, and then provide + more details on three potential troubleshooting items: + + 1 - using a model from a custom/private path - and 'inserting' directly into the model_repo lookup + 2 - identifying the prompt wrapper template + 3 - customizing a new prompt wrapper + +""" + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt +from llmware.configs import LLMWareConfig + +# Create model card and register in the ModelCatalog + +""" Sample OpenVino Model Card template + + model_card_dict = {"model_name": "phi-3-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml"], + "link": "https://huggingface.co/llmware/phi-3-ov"}, +""" + +""" Sample ONNX Model Card template + +model_card_dict = {"model_name": "phi-3-onnx", "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/phi-3-onnx"}, +""" + +# create the model card dictionary manually using the templates above as guides, e.g., +model_card_dict = {"model_name": "my_model", "insert other params from above...": []} + +# this is the key step - registering the model card - add as a first line in any script/example +ModelCatalog().register_new_model_card(model_card_dict) + +# once the model is registered in the catalog, it can then be accessed anytime by name, e.g., +model = ModelCatalog().load_model("my_model") +response = model.inference("What is ...") + +# or if using in conjunction with building a RAG prompt +prompter = Prompt().load_model("my_model") + +""" Issue # 1 - Models in local/custom path + + If you have the model in a local/custom path, then the easiest thing to do is to copy/move manually to + /llmware_data/model_repo/{{my_model_name}}/ and place the model components in this path. +""" + +# lookup model repo path +model_path = LLMWareConfig().get_model_repo_path() +print("local model path: ", model_path) + +# You can manually put the model components in a folder called "model_name" at the model repo path, and +# 'lookups' will all work. + +""" Issue # 2 - How do I figure out the prompt template? + + Below is a list of the prompt wrapper lookups that covers most of the common models: + + # standard used in most llmware models - bling, dragon and slim + "human_bot": {"main_start": ": ", "main_stop": "\n", "start_llm_response": ":"}, + + # commonly used by llama2 and mistral + "": {"main_start": "", "main_stop": "", "start_llm_response": ""}, + + "hf_chat": {"system_start": "<|im_start|>system\n", "system_stop": "<|im_end|>\n", + "main_start": "<|im_start|>user", "main_stop": "<|im_end|>\n", + "start_llm_response": "<|im_start|>assistant"}, + + "open_chat": {"main_start": "GPT4 User: ", "main_stop": "<|endofturn|>", + "start_llm_response": "GPT4 Assistant:"}, + + "alpaca": {"main_start": "### Instruction: ", "main_stop": "\n", + "start_llm_response": "### Response: "}, + + "chat_ml": {"system_start": "<|im_start|>system", "system_stop": "<|im_end|>\n", + "main_start": "<|im_start|>user", "main_stop": "<|im_end|>\n", + "start_llm_response": "<|im_start|>assistant"}, + + "phi_3": {"system_start": "<|system|>\n", "system_stop": "<|end|>\n", + "main_start": "<|user|>\n", "main_stop": "<|end|>\n", "start_llm_response": "<|assistant|>"}, + + "llama_3_chat": {"system_start": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n", + "system_stop": "<|eot_id|>", + "main_start": "<|start_header_id|>user>|end_header_id|>\n", + "main_stop": "<|eot_id|>", + "start_llm_response": "<|start_header_id|>assistant<|end_header_id|>\n"}, + + "tiny_llama_chat": {"system_start": "<|system|>", "system_stop": "", + "main_start": "<|user|>", "main_stop": "", + "start_llm_response": "<|assistant|>"}, + + "stablelm_zephyr_chat": {"system_start": "", "system_stop": "", + "main_start": "<|user|>", "main_stop": "<|endoftext|>\n", + "start_llm_response": "<|assistant|>"}, + + "google_gemma_chat": {"system_start": "", "system_stop": "", + "main_start": "user\n", + "main_stop": "\n", + "start_llm_response": "model"}, + + "vicuna_chat": {"system_start": "", "system_stop": "", + "main_start": "USER: ", "main_stop": "", + "start_llm_response": " ASSISTANT:"} + +""" + +# if none of these templates work, then you can also register a new prompt template +ModelCatalog().register_new_finetune_wrapper("my_new_template", + main_start="", + main_stop="", + llm_start="", + system_start=" use_top: + output = output[0:use_top] + + for i, results in enumerate(output): + print("semantic ranking results: ", i, results["rerank_score"], results["text"]) + + sources = prompter.add_source_query_results(output) + responses = prompter.prompt_with_source(query,prompt_name="default_with_context") + + for i, resp in enumerate(responses): + print("\nllm answers: ", i, resp) + + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + rag_in_memory_with_reranker() diff --git a/solutions/openvino/README.md b/solutions/openvino/README.md new file mode 100644 index 0000000..b13ba5d --- /dev/null +++ b/solutions/openvino/README.md @@ -0,0 +1,233 @@ + +# Developer Guide for Building AI Applications with LLMWare and OpenVINO™ + +## Table of Contents + +* [Introduction](#introduction) +* [Prerequisites and Installation](#prerequisites-and-installation) + * [Setup and Install LLMWare and OpenVINO](#setup-and-install-llmware-and-openvino) +* [Model Catalog Integration](#model-catalog-integration) + * [Loading OpenVINO Models](#loading-openvino-models) + * [Performance Benefits](#performance-benefits) + * [Available Model Families](#available-model-families) +* [Quick Start Examples](#quick-start-examples) +* [OpenVINO Model Configuration and Optimizations](#openvino-model-configuration-and-optimizations) + * [Temperature and Sampling](#temperature-and-sampling) + * [Device Placement and Performance Tuning](#device-placement-and-performance-tuning) + * [OpenVINO Model Loading Phases](#openvino-model-loading-phases) + +## Introduction + +This guide provides a concise walkthrough for developers on using [LLMWare](https://llmware-ai.github.io/llmware/getting_started) with [OpenVINO](https://docs.openvino.ai/2025/index.html) to build high-performance AI applications. LLMWare's integration with OpenVINO is designed to be seamless and straightforward. The core principle is that OpenVINO models are treated as **"drop-in" replacements** for any other model type in `llmware`. This is achieved by loading all models through the standard `ModelCatalog` interface. To use an OpenVINO-optimized model, you simply need to select a model with the `-ov` suffix from the [Model Catalog](https://huggingface.co/llmware/models?search=-ov). + +```python +from llmware.models import ModelCatalog + +# Load any OpenVINO(OV) model by name ending with "-ov" +ov_model = ModelCatalog().load_model("bling-tiny-llama-ov") +``` + +## Prerequisites and Installation + +For detailed system requirements, please see the [Platform Support Guide.](https://llmware-ai.github.io/llmware/getting_started/platforms) + +> [!NOTE] +> For optimal performance, the respective Intel GPU or NPU drivers must be updated. +> Driver installation: [GPU](https://docs.openvino.ai/2025/get-started/install-openvino/configurations/configurations-intel-gpu.html) | [NPU](https://docs.openvino.ai/2025/get-started/install-openvino/configurations/configurations-intel-npu.html) + +#### Setup and Install LLMWare and OpenVINO: +```bash +# (Windows) Set Up Python Virtual Environment and Activate +python -m venv llmware-ov-env +llmware-ov-env\Scripts\activate + +# (Linux) Set Up Python Virtual Environment and Activate +python3 -m venv llmware-ov-env +source llmware-ov-env/bin/activate + +# Install llmware and openvino_genai +pip install llmware openvino_genai +``` + +## Model Catalog Integration + +LLMWare integrated over **100+ OpenVINO and ONNX models** from the [Model Depot](https://medium.com/@darrenoberst/model-depot-9e6625c5fc55) collection into the default model catalog. This extensive collection provides ready-to-use, pre-optimized models for various AI applications. Explore the complete OpenVINO model collection in the [llmware repository on Hugging Face (`-ov` suffix).](https://huggingface.co/llmware/models?search=-ov) + +Because OpenVINO models are loaded through the standard catalog, they are **fully compatible with all other `llmware` features**, including advanced RAG pipelines, agentic workflows, and function-calling with SLIM models. + +### Loading OpenVINO Models +LLMWare introduced the `OVGenerativeModel` class to support models packaged in OpenVINO format. This class provides optimized inference performance particularly beneficial for Intel CPU, GPU and NPU architectures. OpenVINO models are loaded through the standard `ModelCatalog` interface, with model names ending in `-ov` to indicate the OpenVINO format. +- [OpenVINO model collection (`-ov` suffix).](https://huggingface.co/llmware/models?search=-ov) +- [OpenVINO NPU friendly model collection (`npu-ov` suffix).](https://huggingface.co/llmware/models?search=npu-ov) + +```python +from llmware.models import ModelCatalog + +# Get all models whose names end with "-ov" +ov_models = [m["model_name"] + for m in ModelCatalog().list_all_models() + if "-ov" in m["model_name"]] + +print(f"Available OpenVINO models: {ov_models}") # For the latest list see Hugging Face LLMWare OpenVINO model collection (`-ov` suffix) +``` +> [!IMPORTANT] +> The models listed via `ModelCatalog().list_all_models()` use [llmware/model_configs.py](https://github.com/llmware-ai/llmware/blob/main/llmware/model_configs.py) and might not contain all the models. For the latest models availability, see [Hugging Face LLMWare OpenVINO model collection (`-ov` suffix)](https://huggingface.co/llmware/models?search=-ov). + +### Performance Benefits +OpenVINO provides several performance optimizations for LLMWare models: + +- **Model Compilation:** Optimized execution graphs for target hardware automatically. +- **Quantization:** Reduced precision for faster inference. +- **Hardware Acceleration:** OpenVINO is highly optimized for Intel x86 architectures, providing significant performance improvements on CPU, GPU and NPU configurations. + +### Available Model Families + +All models are prepackaged in inference-ready x86-optimized formats, such as OpenVINO and ONNX, quantized with int4, and include smart quantization ratios to mitigate quality impacts (e.g., retaining some parameters at 8-bit). + +* **Leading Generative Models**: leading generative decoder models from 1B — 14B+ parameters in the following leading open source series: Llama 3.2/3.1/3.0/2, Qwen 2.5/2, Mistral 0.3/0.2/0.1, Phi-3, Gemma-2, Yi 1.5/1.0, StableLM, Tiny Llama and popular and leading fine-tunes including Zephyr, Dolphin, Bling, OpenHermes, Wizard, OpenOrca, Nemo, and Dragon; + +* **Specialized Models**: specialized fine-tuned models in math and programming including: Mathstral, Qwen Code-7B, and CodeGemma; + +* **Multimodal Models**: Qwen2-VL-7B, Qwen2-VL-2B, Llama 3.2 11B vision designed for edge deployment of vision+text -> text models; + +* **BLING Models**: Small CPU-optimized models (1B-3B parameters); + +* **DRAGON Models**: Larger RAG-optimized models; + +* **Function-Calling Models**: specialized function-calling SLIM models for multi-model, multi-step agent-based workflows; and + +* **Encoders**: embedding models, rerankers, and classifiers. + +* **Custom Model Integration**: To add your own OpenVINO models to LLMWare, see [examples/Models/adding_openvino_or_onnx_model.py](https://github.com/llmware-ai/llmware/blob/main/examples/Models/adding_openvino_or_onnx_model.py) + +> [!NOTE] +> The models are all in open source, licensed on permissive terms consistent with the terms of the underlying models, and made available as a resource to the wider community to use in their own deployments. + +## Quick Start Examples + +Explore a wide range of [examples in the llmware repository](https://llmware-ai.github.io/llmware/examples). Because OpenVINO models are loaded through the standard catalog, they are **fully compatible with all other `llmware` features**, including advanced RAG pipelines, agentic workflows, and function-calling with SLIM models. + +- **Core OpenVINO Examples**: See [examples/Models/using_openvino_models.py](https://github.com/llmware-ai/llmware/blob/main/examples/Models/using_openvino_models.py) + ```python + from llmware.models import ModelCatalog + + # --------------------------- + # Basic Inference Example + # --------------------------- + # Load an OpenVINO-optimized model + model = ModelCatalog().load_model("bling-tiny-llama-ov") + + # Perform inference + response = model.inference("What are the key benefits of using OpenVINO?") + print(f"Response: {response}") + + # --------------------------- + # Sentiment Analysis Example + # --------------------------- + # Load the OpenVINO optimized sentiment analysis model + sentiment_model = ModelCatalog().load_model("slim-sentiment-ov") + + # Analyze sentiment + result = sentiment_model.function_call("I love using LLMWare with OpenVINO! The performance is amazing.") + print(f"Sentiment Analysis Result: {result}") + + # --------------------------- + # Information Extraction Example + # --------------------------- + # Load the OpenVINO optimized information extraction model + extraction_model = ModelCatalog().load_model("slim-extract-ov") + + # Extract key information + text = "The invoice total is $1,234.56 and the due date is 2024-12-31." + extracted_info = extraction_model.function_call(text, function="extract", params=["invoice total", "due date"]) + print(f"Extracted Information: {extracted_info}") + ``` +- **Advanced Multimedia Bot**: A multi-threaded application using multiple OpenVINO models on different hardware (CPU, GPU, NPU) simultaneously. See [examples/UI/multimedia_bot.py](https://github.com/llmware-ai/llmware/blob/main/examples/UI/multimedia_bot.py) +- **Fast Start Examples**: [Learn llmware through examples.](https://github.com/llmware-ai/llmware/tree/main/fast_start) + - RAG Pipelines: See [fast_start/rag/](https://github.com/llmware-ai/llmware/tree/main/fast_start/rag) + - Agentic workflows: See [fast_start/agents/](https://github.com/llmware-ai/llmware/tree/main/fast_start/agents) + +## OpenVINO Model Configuration and Optimizations + +### Temperature and Sampling + +You can control the behavior of OpenVINO models using standard `llmware` parameters passed during the `load_model` call. + +- **`temperature`**: Controls randomness in the output. A value of `0.0` is deterministic. +- **`sample`**: A boolean that enables or disables sampling. +- **`max_output`**: An integer to limit the length of the generated response. + +```python +# Example of loading a model with specific sampling parameters +model = ModelCatalog().load_model( + "bling-tiny-llama-ov", + temperature=0.0, + sample=False, + max_output=256, +) +``` + +### Device Placement and Performance Tuning +OpenVINO models automatically detect and utilize available hardware: +- **CPU:** Default fallback, optimized for Intel architectures. +- **GPU:** Automatically used when an Intel GPU (integrated or discrete) is available. +- **NPU:** Supported on the Intel Core Ultra processors (codename "Meteor Lake" and newer) for sustained, low-power AI workloads. Must be targeted explicitly by setting `device="NPU"`. + +You can fine-tune OpenVINO performance using the `OVConfig` class. This is particularly useful for specifying device placement or adjusting performance hints. + +```python +from llmware.models import ModelCatalog +from llmware.configs import OVConfig + +# Option 1: Use OVConfig to set a global default device +OVConfig().set_config("device", "CPU") + +# Option 2: Specify the device directly when loading a model +npu_model = ModelCatalog().load_model("slim-topics-npu-ov", device="NPU") +``` + +The available configuration options in `OVConfig` are: + +| Parameter | Type | Default Value | Description | +| ------------------------- | ------- | -------------------- | ----------------------------------------------------------------| +| `device` | `str` | `"GPU"` | The device to run the model on (`"CPU"` or `"GPU"` or `"NPU"`). | +| `use_ov_tokenizer` | `bool` | `False` | Whether to use the OpenVINO™ tokenizer. | +| `generation_version` | `str` | `"ov_genai_pip"` | The generation version to use. | +| `use_gpu_if_available` | `bool` | `True` | If `True`, automatically uses the GPU if available. | +| `cache` | `bool` | `True` | Enables caching of models to optimize subsequent loads. | +| `cache_with_model` | `bool` | `True` | If `True`, caches with the model. | +| `cache_custom_path` | `str` | `""` | A custom path for caching models. | +| `apply_performance_hints` | `bool` | `True` | Applies performance hints for GPU. | +| `verbose_mode` | `bool` | `False` | Enables verbose logging for debugging. | +| `get_token_counts` | `bool` | `True` | Whether to retrieve token counts during inference. | + + +You can also set GPU-specific performance hints: +```python +# Example: Set the model priority to high +OVConfig().set_gpu_hint("MODEL_PRIORITY", "HIGH") +``` +- Supported GPU hints include: `MODEL_PRIORITY`, `GPU_HOST_TASK_PRIORITY`, `GPU_QUEUE_THROTTLE`, `GPU_QUEUE_PRIORITY`. +- For more details, you can view the source code for the `OVConfig` class in [`llmware/configs.py`](https://github.com/llmware-ai/llmware/blob/main/llmware/configs.py#L537). + +### OpenVINO Model Loading Phases +The OpenVINO model loading follows these phases when `ModelCatalog().load_model("..."),` is executed: +1. **Download**: Checks local copy of the model; if missing, downloads from Hugging Face. +2. **First Inference**: Compiles model, saves compiled artifacts as cache in model directory. +3. **Subsequent Inferences**: Reuses cached artifacts. No recompilation is needed as the cache contains the optimized model representation, making subsequent model loads much faster. + +The caching behavior can be customized through `OVConfig` methods if needed, such as disabling caching with `OVConfig().set_config("cache", False)` or setting a custom cache path. + +Below is a sample to inspect model storage locations and verify downloaded OpenVINO models and caches locally. +```python +from llmware.configs import LLMWareConfig +from llmware.models import ModelCatalog + +# Print the general model repository path +model_repo_path = LLMWareConfig.get_model_repo_path() +print(f"Model repository path: {model_repo_path}") + +# Load model (downloads if missing, compiles and caches on first use, uses caches for subsequent) +ov_model = ModelCatalog().load_model("bling-tiny-llama-ov") +print("Loaded model path:", ov_model.model_repo_path) +``` diff --git a/solutions/openvino/adding_openvino_or_onnx_model.py b/solutions/openvino/adding_openvino_or_onnx_model.py new file mode 100644 index 0000000..659f9c3 --- /dev/null +++ b/solutions/openvino/adding_openvino_or_onnx_model.py @@ -0,0 +1,144 @@ + +""" This example shows how to add a custom or private OpenVino or ONNX model to the llmware model catalog. + + Over the next few releases, we will be expanding the default ModelCatalog considerably, but for the time + being, please feel free to follow the steps below to build your own custom catalog. + + We show below templates for the model card dictionaries - most of which is fairly easy to build for a given + model. + + We highlight both the main step - which is a simple one-liner to register the model, and then provide + more details on three potential troubleshooting items: + + 1 - using a model from a custom/private path - and 'inserting' directly into the model_repo lookup + 2 - identifying the prompt wrapper template + 3 - customizing a new prompt wrapper + +""" + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt +from llmware.configs import LLMWareConfig + +# Create model card and register in the ModelCatalog + +""" Sample OpenVino Model Card template + + model_card_dict = {"model_name": "phi-3-ov", "model_family": "OVGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3-ov", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3-ov", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["openvino_model.xml"], + "link": "https://huggingface.co/llmware/phi-3-ov"}, +""" + +""" Sample ONNX Model Card template + +model_card_dict = {"model_name": "phi-3-onnx", "model_family": "ONNXGenerativeModel", + "model_category": "generative_local", "display_name": "phi-3-onnx", + "model_location": "llmware_repo", + "context_window": 4096, "instruction_following": False, "prompt_wrapper": "phi_3", + "temperature": 0.0, "sample_default": False, "trailing_space": "", + "tokenizer_local": "tokenizer_phi3.json", + "hf_repo": "llmware/phi-3-onnx", + "custom_model_files": [], "custom_model_repo": "", + "fetch": {"snapshot": True, "module": "llmware.models", "method": "pull_snapshot_from_hf"}, + "validation_files": ["model.onnx", "model.onnx.data"], + "link": "https://huggingface.co/llmware/phi-3-onnx"}, +""" + +# create the model card dictionary manually using the templates above as guides, e.g., +model_card_dict = {"model_name": "my_model", "insert other params from above...": []} + +# this is the key step - registering the model card - add as a first line in any script/example +ModelCatalog().register_new_model_card(model_card_dict) + +# once the model is registered in the catalog, it can then be accessed anytime by name, e.g., +model = ModelCatalog().load_model("my_model") +response = model.inference("What is ...") + +# or if using in conjunction with building a RAG prompt +prompter = Prompt().load_model("my_model") + +""" Issue # 1 - Models in local/custom path + + If you have the model in a local/custom path, then the easiest thing to do is to copy/move manually to + /llmware_data/model_repo/{{my_model_name}}/ and place the model components in this path. +""" + +# lookup model repo path +model_path = LLMWareConfig().get_model_repo_path() +print("local model path: ", model_path) + +# You can manually put the model components in a folder called "model_name" at the model repo path, and +# 'lookups' will all work. + +""" Issue # 2 - How do I figure out the prompt template? + + Below is a list of the prompt wrapper lookups that covers most of the common models: + + # standard used in most llmware models - bling, dragon and slim + "human_bot": {"main_start": ": ", "main_stop": "\n", "start_llm_response": ":"}, + + # commonly used by llama2 and mistral + "": {"main_start": "", "main_stop": "", "start_llm_response": ""}, + + "hf_chat": {"system_start": "<|im_start|>system\n", "system_stop": "<|im_end|>\n", + "main_start": "<|im_start|>user", "main_stop": "<|im_end|>\n", + "start_llm_response": "<|im_start|>assistant"}, + + "open_chat": {"main_start": "GPT4 User: ", "main_stop": "<|endofturn|>", + "start_llm_response": "GPT4 Assistant:"}, + + "alpaca": {"main_start": "### Instruction: ", "main_stop": "\n", + "start_llm_response": "### Response: "}, + + "chat_ml": {"system_start": "<|im_start|>system", "system_stop": "<|im_end|>\n", + "main_start": "<|im_start|>user", "main_stop": "<|im_end|>\n", + "start_llm_response": "<|im_start|>assistant"}, + + "phi_3": {"system_start": "<|system|>\n", "system_stop": "<|end|>\n", + "main_start": "<|user|>\n", "main_stop": "<|end|>\n", "start_llm_response": "<|assistant|>"}, + + "llama_3_chat": {"system_start": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n", + "system_stop": "<|eot_id|>", + "main_start": "<|start_header_id|>user>|end_header_id|>\n", + "main_stop": "<|eot_id|>", + "start_llm_response": "<|start_header_id|>assistant<|end_header_id|>\n"}, + + "tiny_llama_chat": {"system_start": "<|system|>", "system_stop": "", + "main_start": "<|user|>", "main_stop": "", + "start_llm_response": "<|assistant|>"}, + + "stablelm_zephyr_chat": {"system_start": "", "system_stop": "", + "main_start": "<|user|>", "main_stop": "<|endoftext|>\n", + "start_llm_response": "<|assistant|>"}, + + "google_gemma_chat": {"system_start": "", "system_stop": "", + "main_start": "user\n", + "main_stop": "\n", + "start_llm_response": "model"}, + + "vicuna_chat": {"system_start": "", "system_stop": "", + "main_start": "USER: ", "main_stop": "", + "start_llm_response": " ASSISTANT:"} + +""" + +# if none of these templates work, then you can also register a new prompt template +ModelCatalog().register_new_finetune_wrapper("my_new_template", + main_start="", + main_stop="", + llm_start="", + system_start=" text generation output. + + Prereqs: + -- pip install openvino_genai + -- pip install PIL (Pillow library for image preparation) + + If you want to use the web streamer: + -- pip install pywebio + + """ + +from llmware.models import ModelCatalog +import openvino_genai as ovg + +try: + from pywebio.output import put_text +except: + pass + + +# this is the default streamer included in the OVGenerativeModel class - +# if no streamer explicitly passed, then this will be used + +def ov_default_streamer(x): + print(x, end="", flush=True) + return ovg.StreamingStatus.RUNNING + +# here is a simple example that will stream the text to local host +# to run this example: `pip install pywebio` + +def web_streamer(x): + put_text(x,inline=True) + return ovg.StreamingStatus.RUNNING + +# supported OpenVINO vision models in llmware: +# -- qwen2.5-vl-3b-ov +# -- gemma-3-4b-ov +# -- phi-3.5-vision-ov +# -- phi-4-mm-ov + +model = ModelCatalog().load_model("phi-4-mm-ov", + max_output=500, device="GPU") + +image_path = "C:\\Users\\path\\to\\image_file" + +prompt = "Describe this image." + +import time +t0=time.time() + +response = model.stream(prompt, image_path, streamer=ov_default_streamer) + +t1= time.time() + + +print("\n\ncompleted streaming response - ", response) +print("\ntime taken: ", t1-t0) + + diff --git a/solutions/openvino/using_local_foundry_models.py b/solutions/openvino/using_local_foundry_models.py new file mode 100644 index 0000000..44b19a3 --- /dev/null +++ b/solutions/openvino/using_local_foundry_models.py @@ -0,0 +1,38 @@ + +""" This example shows how to use Windows Local Foundry models. + + pre-reqs: + + 1. install local foundry, e.g., `winget install Microsoft.FoundryLocal` + 2. pip3 install foundry-local-sdk + 3. pip3 install openai (openai api used, but not openai model - all runs locally) +""" + +from llmware.models import ModelCatalog, WindowsLocalFoundryHandler + +# activate the connection and poll foundry-local instance for available models +foundry_handler = WindowsLocalFoundryHandler() +cat = foundry_handler.activate_catalog(True) + +# foundry local models now added to the catalog +foundry_models = ModelCatalog().list_models_by_type("WindowsLocalFoundryModel") +for i, mod in enumerate(foundry_models): + print("--foundry model - ", mod) + +all_models = ModelCatalog().list_all_models() + +# load foundry model like any other in llmware +# note: this example was used on a Windows Intel x86 Lunar Lake +# -- different platforms will have different supported models + +m1 = "Phi-3.5-mini-instruct-openvino-gpu:1-foundry" +m2 = "qwen2.5-0.5b-instruct-generic-cpu:4-foundry" +model = ModelCatalog().load_model(m1, max_output=500) + +for token in model.stream("What are the best sites to see in Rome?"): + print(token, end="") + +# stop foundry local server when done +WindowsLocalFoundryHandler().stop_server() + + diff --git a/solutions/openvino/using_openvino_classifier_model.py b/solutions/openvino/using_openvino_classifier_model.py new file mode 100644 index 0000000..c330f55 --- /dev/null +++ b/solutions/openvino/using_openvino_classifier_model.py @@ -0,0 +1,25 @@ + +""" Using OpenVINO classifier-based models - illustrates how to get started using + classifier encoding models with OpenVINO." + + `pip3 install openvino` + +""" + +from llmware.models import ModelCatalog + +# classifies the likelihood of a prompt injection +cl_name = "protectai-prompt-injection-ov" + +# e.g., other examples: +# -- "unitary-toxic-roberta-ov" - classifies potential toxicity +# -- "valurank-bias-ov" - classifies potential bias + +classifier_model = ModelCatalog().load_model(cl_name) + +text = "The sun is shining in New York today." + +response = classifier_model.classify(text) + +print("--response: ", response) + diff --git a/solutions/openvino/using_openvino_embedding_model.py b/solutions/openvino/using_openvino_embedding_model.py new file mode 100644 index 0000000..a000bdb --- /dev/null +++ b/solutions/openvino/using_openvino_embedding_model.py @@ -0,0 +1,26 @@ + +""" OpenVINO embedding models - this example shows how to use OpenVINO embedding + models which can be prepared in batch, and used in conjunction with vector databases + or other vector semantic search retrieval applications. + + Prerequisites: + -- pip3 install openvino + +""" + +from llmware.models import ModelCatalog + +# industry-bert-contracts-ov +# industry-bert-insurance-ov +# all-mini-lm-l6-v2-ov +# all-mpnet-base-v2-ov + +model = ModelCatalog().load_model("industry-bert-contracts-ov") + +text = "We are at the airport waiting for our flight." +text2 = "The airport is boring, but OK to work from." +text3 = "I am looking forward to our trip." + +embedding = model.embedding([text, text2,text3]) + +print("--test: embedding - ", embedding.shape, embedding) diff --git a/solutions/openvino/using_openvino_models.py b/solutions/openvino/using_openvino_models.py new file mode 100644 index 0000000..de46444 --- /dev/null +++ b/solutions/openvino/using_openvino_models.py @@ -0,0 +1,97 @@ + +""" Starting with llmware 0.3.7, we have integrated support for OpenVino Generative models. + + To get started: + + `pip install openvino` + `pip install openvino_genai` + + Openvino is supported on a wide range of platforms (including Windows, Linux, Mac OS), and is highly + optimized for Intel x86 architectures - both CPU and GPU. + + The intent is for OpenVino models to be "drop in" replacements for Pytorch or GGUF models by simply + replacing the model with the OpenVino equivalent - usually indicated by an 'ov' at the end of the model name + + """ + +from llmware.models import ModelCatalog + +from importlib import util +if not util.find_spec("openvino"): + print("\nto run this example, you need to install openvino first, e.g., pip3 install openvino") + +if not util.find_spec("openvino_genai"): + print("\nto run this example, you need to install openvino_genai first, e.g., pip3 install openvino_genai") + + +# as of llmware 0.3.8, we have integrated the Model Depot collection into the default llmware model catalog +# please check out home page in Huggingface for a complete view of the collection +# https://www.huggingface.co/llmware + +# to add your own OpenVino models, please see the example 'adding_openvino_or_onnx_model.py' + + +def getting_started(): + + model = ModelCatalog().load_model("bling-tiny-llama-ov", temperature=0.0, sample=False, + max_output=100) + + query= "What was Microsoft's revenue in the 3rd quarter?" + + context = ("Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n") + + response = model.inference(query ,add_context=context) + + print(f"\ngetting_started example - query - {query}") + print("getting_started example - response: ", response) + + return response + + +def sentiment_analysis(): + + model = ModelCatalog().load_model("slim-sentiment-ov", temperature=0.0,sample=False) + + text = ("The poor earnings results along with the worrisome guidance on the future has dampened " + "expectations and put a lot of pressure on the share price.") + + response = model.function_call(text) + + print(f"\nsentiment_analysis - {response}") + + return response + + +def extract_info(): + + model = ModelCatalog().load_model("slim-extract-tiny-ov", temperature=0.0, sample=False) + + text = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + + response = model.function_call(text,function="extract", params=["termination fee"]) + + print(f"\nextract_info - {response}") + + return response + + +if __name__ == "__main__": + + getting_started() + sentiment_analysis() + extract_info() + diff --git a/solutions/openvino/using_openvino_npu_models.py b/solutions/openvino/using_openvino_npu_models.py new file mode 100644 index 0000000..776d194 --- /dev/null +++ b/solutions/openvino/using_openvino_npu_models.py @@ -0,0 +1,40 @@ + +""" This example illustrates how to run models on an Intel NPU using OpenVINO, using + a simple callback streaming function. + + Prereqs: + -- pip install openvino_genai + + """ + +from llmware.models import ModelCatalog +import openvino_genai as ovg + + +# this is the default streamer included in the OVGenerativeModel class - +# if no streamer explicitly passed, then this will be used + +def ov_default_streamer(x): + print(x, end="", flush=True) + return ovg.StreamingStatus.RUNNING + +# lists all intel npu optimized models in the catalog +npu_models = ModelCatalog().list_intel_npu_optimized_models() + + +for i, model in enumerate(npu_models): + print("--intel npu models - ", i, model.get("model_name", "")) + +# use any of these models, and the NPU will automatically be set as the device +# the device can also be manually by adding: device="NPU" to the load_model call + +model = ModelCatalog().load_model("llama-3.2-1b-instruct-npu-ov", + max_output=500) + +prompt = "What are the best sites to see in France?" + +# streamer is None by default, pass custom streaming function here +response = model.stream(prompt, streamer=ov_default_streamer) + +print("\n\ncompleted streaming response - ", response) + diff --git a/solutions/openvino/using_openvino_reranker_model.py b/solutions/openvino/using_openvino_reranker_model.py new file mode 100644 index 0000000..1b479af --- /dev/null +++ b/solutions/openvino/using_openvino_reranker_model.py @@ -0,0 +1,71 @@ + +""" openvino reranker model - this example shows how to use openvino reranker model - + it is modeled directly off other reranker example in the repository - + + please note that you should import openvino to run this example, e.g., + + -- `pip install openvino` + +""" + +import os + +from llmware.parsers import Parser +from llmware.models import ModelCatalog +from llmware.prompts import Prompt +from llmware.setup import Setup + +def rag_in_memory_with_reranker(): + + """ Executes a rag process in memory using semantic reranker model and bling-phi-3-gguf to answer the question. """ + + query = "What is the annual rate of the executive's base salary?" + + sample_files_path = Setup().load_sample_files(over_write=False) + contracts_path = os.path.join(sample_files_path, "Agreements") + + files = os.listdir(contracts_path) + + # will use two models for the example - reranker + a 'question-answer' rag llm + + # reranker examples: + # -- jina-reranker-v1-turbo-en-ov + # -- jina-reranker-v1-tiny-en-ov + + # use ov reranker model to find the most relevant parts of the text + reranker_model = ModelCatalog().load_model("jina-reranker-v1-tiny-en-ov") + + # use small ov generative model to review and answer the question + prompter = Prompt().load_model("bling-tiny-llama-ov", temperature=0.0, sample=False) + + for i, doc in enumerate(files): + + if doc.endswith(".pdf"): + + print("\nPROCESSING: ", i, doc) + + parser_output = Parser().parse_one(contracts_path,doc,save_history=False) + + output = reranker_model.rank(query,parser_output,top_n=10, relevance_threshold=0.25) + + use_top = 3 + if len(output) > use_top: + output = output[0:use_top] + + for i, results in enumerate(output): + print("semantic ranking results: ", i, results["rerank_score"], results["text"]) + + sources = prompter.add_source_query_results(output) + responses = prompter.prompt_with_source(query,prompt_name="default_with_context") + + for i, resp in enumerate(responses): + print("\nllm answers: ", i, resp) + + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + rag_in_memory_with_reranker() diff --git a/solutions/openvino/using_openvino_streamer.py b/solutions/openvino/using_openvino_streamer.py new file mode 100644 index 0000000..4f6a11a --- /dev/null +++ b/solutions/openvino/using_openvino_streamer.py @@ -0,0 +1,50 @@ + +""" This example illustrates the use of streaming with OpenVINO models, which + use a callback streaming function, rather than a generator - the example + shows how to stream text to console using the default streamer, and + how to modify and create a custom streamer function. + + Prereqs: + -- pip install openvino_genai + + If you want to use the web streamer: + -- pip install pywebio + + """ + +from llmware.models import ModelCatalog +import openvino_genai as ovg + +try: + from pywebio.output import put_text +except: + pass + + +# this is the default streamer included in the OVGenerativeModel class - +# if no streamer explicitly passed, then this will be used + +def ov_default_streamer(x): + print(x, end="", flush=True) + return ovg.StreamingStatus.RUNNING + +# here is a simple example that will stream the text to local host +# to run this example: `pip install pywebio` + +def web_streamer(x): + put_text(x,inline=True) + return ovg.StreamingStatus.RUNNING + + +model = ModelCatalog().load_model("llama-3.2-3b-instruct-ov", + max_output=500) + +prompt = "What are the best sites to see in France?" + +# streamer is None by default, pass custom streaming function here +response = model.stream(prompt, streamer=ov_default_streamer) + +print("\n\ncompleted streaming response - ", response) + + + diff --git a/solutions/rag/README.md b/solutions/rag/README.md new file mode 100644 index 0000000..643973b --- /dev/null +++ b/solutions/rag/README.md @@ -0,0 +1,93 @@ + + +Fast Start: Learning RAG with llmware through examples +=============== + +**Welcome to llmware!** + +Set up + +`pip3 install llmware` or `pip3 install 'llmware[full]'` or, if you prefer clone the github repo locally, e.g., `git clone git@github.com:llmware-ai/llmware.git`. If you clone the repo, then we would recommend that you run the `welcome_to_llmware.sh` or `welcome_to_llmware_windows.sh` scripts to install all of the dependencies. + +Note: starting in llmware>=0.3.0, we offer two pip install options. If you use the standard `pip3 install llmware`, then you will need to add a few additional pip3 installs to run examples 2 and 5 below, specifically: + + `pip3 install torch` + `pip3 install transformers` + +Platforms: +- Mac M1/M2/M3, Windows, Linux (Ubuntu 20 or Ubuntu 22 preferred) +- RAM: 16 GB minimum +- Python 3.9, 3.10, 3.11, 3.12 +- Pull the latest version of llmware +- Please note that we have updated the examples from the original versions, to use new features in llmware, so there may be minor differences with the videos, which are annotated in the comments in each example. + +There are 9 examples, designed to be used step-by-step, but each is self-contained, +so you can feel free to jump into any of the examples, in any order, that you prefer. + +Each example has been designed to be "copy-paste" and RUN with lots of helpful comments and explanations embedded in the code samples. + +Please check out our [Fast Start Youtube tutorials](https://www.youtube.com/playlist?list=PL1-dn33KwsmD7SB9iSO6vx4ZLRAWea1DB) that walk through each example below. + +Examples: + +**Section I - Learning the Main Components** +1. **Library** - parse, text chunk, and index to convert a "pile of files" into an AI-ready knowledge-base. [Video](https://youtu.be/2xDefZ4oBOM?si=8vRCvqj0-HG3zc4c) + +2. **Embeddings** - apply an embedding model to the Library, store vectors, and start enabling natural language queries. [Video](https://youtu.be/xQEk6ohvfV0?si=B3X25ZsAZfW4AR_3) + +3. **Prompts** & **Model Catalog** - start running inferences and building prompts. [Video](https://youtu.be/swiu4oBVfbA?si=0IVmLhiiYS3-pMIg) + +**Section II - Connecting Knowledge with Prompts - 3 scenarios** + +4. **RAG with Text Query** - start integrating documents into prompts. [Video](https://youtu.be/6oALi67HP7U?si=pAbvio4ULXTIXKdL) + +5. **RAG with Semantic Query** - use natural language queries on documents and integrate with prompts. [Video](https://youtu.be/XT4kIXA9H3Q?si=EBCAxVXBt5vgYY8s) + +6. **RAG with more complex retrieval** - start integrating more complex retrieval patterns. [Video](https://youtu.be/G1Q6Ar8THbo?si=vIVAv35uXAcnaUJy) + +**Section III - Function Calls & Agents** + +7. **Function Calls** - move beyond 'question-answer' prompting and start prompting with function calls. + +8. **Agents** - the power of function calls is the ability to integrate model function calls as 'tools' available to an agent orchestrator. [Video](https://youtu.be/cQfdaTcmBpY?si=pMWQj0qpPBVRmm34) + +9. **Function Calls with Web Services** - one of the most exciting use cases is the ability to combine function calls with web services. [Video](https://youtu.be/l0jzsg1_Ik0?si=ifwxVi_Z6I_hNtcf) + +After completing these 9 examples, you should have a good foundation and set of recipes to start +exploring the other 100+ examples in the /examples folder, and build more sophisticated +LLM-based applications. + +**Models** + - All of these examples are optimized for using local CPU-based models, primarily BLING, DRAGON and SLIM models. + - If you want to substitute for any other model in the catalog, it is generally as easy as + switching the model_name. If the model requires API keys, we show in the examples how to pass those keys as an + environment variable. + +**Collection Databases** + - Our parsers are optimized to index text chunks directly into a persistent data store. + - For Fast Start, we will use "sqlite" which is an embedded database, requiring no install + - For more scalable deployment, we would recommend either "mongo" or "postgres" + - Install instructions for "mongo" and "postgres" are provided in docker-compose files in the repository + +**Vector Databases** + - For Fast Start, we will use a no-install vector db (in Examples 2 and 5 specifically). + - There are 4 no-install options supported, but depending upon your enviroment, you may need to pip3 install the corresponding vector db python sdk, eg.: + + - chromadb: `pip3 install chromadb` + - milvus lite: `pip3 install pymilvus` (Mac and Linux only) + - faiss: `pip3 install faiss` + - lancedb: `pip3 install lancedb` + + - For more scalable deployment, we would recommend installing one of 9 supported vector databases, + including Milvus, PGVector (Postgres), Redis, Qdrant, Neo4j, Mongo-Atlas, Chroma, LanceDB, or Pinecone. + - Install instructions provided in "examples/Embedding" for specific db, as well as docker-compose scripts. + +**Local Private** + - All of the processing will take place locally on your laptop. + +*This is an ongoing initiative to provide easy-to-get-started tutorials - we welcome and encourage feedback, as well +as contributions with examples and other tips for helping others on their LLM application journeys!* + +**Let's get started!** + + diff --git a/solutions/rag/example-1-create_first_library.py b/solutions/rag/example-1-create_first_library.py new file mode 100644 index 0000000..0ab0446 --- /dev/null +++ b/solutions/rag/example-1-create_first_library.py @@ -0,0 +1,129 @@ + +""" Fast Start Example #1 - Library - converting document files into an indexed knowledge collection. + + In this example, we will illustrate a basic recipe for completing the following steps: + + 1. Create a library as an organizing construct for your knowledge-base + 2. Download sample files for a Fast Start - easy to 'swap out' and replace with your own files + 3. Use library.add_files method to automatically parse, text chunk and index the documents + 4. Run a basic text query against your new Library +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def parsing_documents_into_library(library_name, sample_folder): + + print(f"\nExample - Parsing Files into Library") + + # create new library + print (f"\nStep 1 - creating library {library_name}") + library = Library().create_new_library(library_name) + + # load the llmware sample files + # -- note: if you have used this example previously, UN-Resolutions-500 is new path + # -- to pull updated sample files, set: 'over_write=True' + + sample_files_path = Setup().load_sample_files(over_write=False) + print (f"Step 2 - loading the llmware sample files and saving at: {sample_files_path}") + + # note: to replace with your own documents, just point to a local folder path that has the documents + ingestion_folder_path = os.path.join(sample_files_path, sample_folder) + + print (f"Step 3 - parsing and indexing files from {ingestion_folder_path}") + + # add files is the key ingestion method - parses, text chunks and indexes all files in folder + # --will automatically route to correct parser based on file extension + # --supported file extensions: .pdf, .pptx, .docx, .xlsx, .csv, .md, .txt, .json, .wav, and .zip, .jpg, .png + + parsing_output = library.add_files(ingestion_folder_path) + + print (f"Step 4 - completed parsing - {parsing_output}") + + # check the updated library card + updated_library_card = library.get_library_card() + doc_count = updated_library_card["documents"] + block_count = updated_library_card["blocks"] + print(f"Step 5 - updated library card - documents - {doc_count} - blocks - {block_count} - {updated_library_card}") + + # check the main folder structure created for the library - check /images to find extracted images + library_path = library.library_main_path + print(f"Step 6 - library artifacts - including extracted images - saved at folder path - {library_path}") + + # use .add_files as many times as needed to build up your library, and/or create different libraries for + # different knowledge bases + + # now, your library is ready to go and you can start to use the library for running queries + # if you are using the "Agreements" library, then a good easy 'hello world' query is "base salary" + # if you are using one of the other sample folders (or your own), then you should adjust the query + + # queries are always created the same way, e.g., instantiate a Query object, and pass a library object + # --within the Query class, there are a variety of useful methods to run different types of queries + + test_query = "base salary" + + print(f"\nStep 7 - running a test query - {test_query}\n") + + query_results = Query(library).text_query(test_query, result_count=10) + + for i, result in enumerate(query_results): + + # note: each result is a dictionary with a wide range of useful keys + # -- we would encourage you to take some time to review each of the keys and the type of metadata available + + # here are a few useful attributes + text = result["text"] + file_source = result["file_source"] + page_number = result["page_num"] + doc_id = result["doc_ID"] + block_id = result["block_ID"] + matches = result["matches"] + + # -- if print to console is too verbose, then pick just a few keys for print + print("query results: ", i, result) + + return parsing_output + + +if __name__ == "__main__": + + # note on sample documents - downloaded by Setup() + # UN-Resolutions-500 is 500 pdf documents + # Invoices is 40 pdf invoice samples + # Agreements is ~15 contract documents + # AgreementsLarge is ~80 contract documents + # FinDocs is ~15 financial annual reports and earnings + # SmallLibrary is a mix of ~10 pdf and office documents + + # optional - set the active DB to be used - by default, it is "mongo" + # if you are just getting started, and have not installed a separate db, select "sqlite" + + # update: as of llmware v0.4.0 (March 2025), the default db is set to sqlite + LLMWareConfig().set_active_db("sqlite") + + # if you want to see a different log view, e.g., see a list of each parsed files 'in progress', + # you can set a different debug mode view anytime + + # debug_mode options - + # 0 - default - shows status manager (useful in large parsing jobs) and errors will be displayed + # 2 - file name only - shows file name being parsed, and errors only + + # for purpose of this example, let's change so we can see file-by-file progress + LLMWareConfig().set_config("debug_mode", 2) + + # this is a list of document folders that will be pulled down by calling Setup() + sample_folders = ["Agreements", "Invoices", "UN-Resolutions-500", "SmallLibrary", "FinDocs", "AgreementsLarge"] + + library_name = "example1_library" + + # select one of the sample folders + selected_folder = sample_folders[0] # e.g., "Agreements" + + # run the example + output = parsing_documents_into_library(library_name, selected_folder) + + diff --git a/solutions/rag/example-2-build_embeddings.py b/solutions/rag/example-2-build_embeddings.py new file mode 100644 index 0000000..600fc6b --- /dev/null +++ b/solutions/rag/example-2-build_embeddings.py @@ -0,0 +1,187 @@ + +""" Fast Start Example #2 - Embeddings - applying an embedding model to enable natural language queries + + In this example, we will show the basic recipe for creating embeddings on a library: + + 1. Create a sample library (see Example #1 for more details) + 2 Select an embedding model + 3. Select a vector db + 4. Install the embeddings + 5. Run a semantic test query + + For purpose of this 'fast start', we will use a no-install option of 'sqlite' as our text collection database + + Note: to run this example with a sentence transformers 'local' open source embedding model, + you may need to install additional dependencies: + + `pip3 install transformers` + `pip3 install torch` + + We would recommend any of the following 'no-install' vector db options: + + -- milvus lite: `pip3 install pymilvus` [available starting in llmware>=0.3.0 on Mac/Linux] + -- chromadb: `pip3 install chromadb` + -- lancedb: `pip3 install lancedb` + -- faiss: `pip3 install faiss` + + -- This same basic recipe will work with any of the vector db and collection db by simply changing the name + +""" + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.resources import Status +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig, MilvusConfig + +from importlib import util + +# generate warnings if key dependencies not involved +if not util.find_spec("torch") or not util.find_spec("transformers"): + print("\nto run this example, with the selected embedding model, please install transformers and torch, e.g., " + "\n`pip install torch`" + "\n`pip install transformers`") + +if not (util.find_spec("chromadb") or util.find_spec("pymilvus") or util.find_spec("lancedb") or util.find_spec("faiss")): + print("\nto run this example, you will need to pip install the vector db drivers. see comments above.") + + +def setup_library(library_name): + + """ Note: this setup_library method is provided to enable a self-contained example to create a test library """ + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # check the embedding status 'before' installing the embedding + embedding_record = library.get_embedding_status() + print("embedding record - before embedding ", embedding_record) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in database + + print("update: Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements"), + chunk_size=400, max_chunk_size=600, smart_chunking=1) + + return library + + +def install_vector_embeddings(library, embedding_model_name): + + """ This method is the core example of installing an embedding on a library. + -- two inputs - (1) a pre-created library object and (2) the name of an embedding model """ + + library_name = library.library_name + vector_db = LLMWareConfig().get_vector_db() + + print(f"\nupdate: Starting the Embedding: " + f"library - {library_name} - " + f"vector_db - {vector_db} - " + f"model - {embedding_model_name}") + + # *** this is the one key line of code to create the embedding *** + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db,batch_size=100) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + # Start using the new vector embeddings with Query + sample_query = "incentive compensation" + print("\n\nupdate: Run a sample semantic/vector query: {}".format(sample_query)) + + # queries are constructed by creating a Query object, and passing a library as input + query_results = Query(library).semantic_query(sample_query, result_count=20) + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # to see all of the dictionary keys returned, uncomment the line below + # print("update: query_results - all - ", i, entries) + + # for display purposes only, we will only show the first 125 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + # lets take a look at the library embedding status again at the end to confirm embeddings were created + embedding_record = library.get_embedding_status() + + print("\nupdate: embedding record - ", embedding_record) + + return 0 + + +if __name__ == "__main__": + + # Fast Start configuration - will use no-install embedded sqlite + # -- if you have installed Mongo or Postgres, then change the .set_active_db accordingly + + LLMWareConfig().set_active_db("sqlite") + + # Select a 'no install' vector db + + # note: starting with llmware>=0.3.0, we support the new milvus lite - you can ignore or comment out if + # using a different vector db - and note: only available on mac/linux + MilvusConfig().set_config("lite", True) + + # select one of: 'milvus' | 'chromadb' | 'lancedb' | 'faiss' + # note: if you run into an error with chromadb, please update to the latest version of llmware==0.3.10 which fixes the issue + LLMWareConfig().set_vector_db("chromadb") + + # Step 1 - this example requires us to have a library created - two options: + + # if you completed example-1 - then load the library you created in that example, e.g., "example1_library" + # uncomment the line below: + # library = Library().load_library("example1_library") + + # alternatively, to use this example as self-contained, then create a new library from scratch: + library = setup_library("example2_library") + + # Step 2 - Select any embedding model in the LLMWare catalog + + # to see a list of the embedding models supported, uncomment the line below and print the list + embedding_models = ModelCatalog().list_embedding_models() + + # for i, models in enumerate(embedding_models): + # print("embedding models: ", i, models) + + # for this first embedding, we will use a very popular and fast sentence transformer + # -- these models require `pip3 install transformers` and `pip3 install torch` + embedding_model = "mini-lm-sbert" + + # note: if you want to swap out "mini-lm-sbert" for Open AI 'text-embedding-ada-002', then: + # 1. you do not need to import transformers or torch + # 2. you should `pip3 install openai` + # 3. you should uncomment these lines: + # embedding_model = "text-embedding-ada-002" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # run the core script + install_vector_embeddings(library, embedding_model) + + + diff --git a/solutions/rag/example-3-prompts_and_models.py b/solutions/rag/example-3-prompts_and_models.py new file mode 100644 index 0000000..6a6e557 --- /dev/null +++ b/solutions/rag/example-3-prompts_and_models.py @@ -0,0 +1,466 @@ + +""" Fast Start Example #3 - Prompts & Model Catalog - how to build prompts and run inferences + + In this example, we will illustrate: + + 1. Discovery - how to discover models in the llmware ModelCatalog + 2. Load Model - how to load a selected model from the catalog + 3. Prompt - how to create a basic prompt and run an inference with the model + + In llmware, we use the same basic formalism for the importing of all models so once you learn the recipe + below, you should be able to import and start inferences on just about any model in the llmware catalog. + + To run GGUF models (generally marked in the catalog as 'GGUFGenerativeModel` model category and + with names that include 'gguf' or 'tool'), then there are no additional dependencies required to start + running local inferences. + + However, pytorch-based models will require additional dependencies to be installed, specifically: + + `pip3 install torch` + `pip3 install transformers` + + To use an OpenAI model, you will need to `pip3 install openai`. + +""" + + +import time +from llmware.prompts import Prompt +from llmware.models import ModelCatalog + + +def hello_world_questions(): + + """ This is a set of useful test questions to do a 'hello world' but there is nothing special about the + questions - please feel free to edit and ask your own queries with your own context passages. + + --if you are using one of the llmware models, please take note that the models have been trained to answer + based on the information provided, so if you ask a question without passing any context passage, then + don't be surprised if the model responds with 'Not Found.' """ + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +def fast_start_prompting (model_name): + + """ This is the main example script - it loads the question list, loads the model and executes the prompts. """ + + t0 = time.time() + + # load in the 'hello world' test questions above + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"], + context=entries["context"], + prompt_name="default_with_context", + temperature=0.30) + + # 'output' is a dictionary with two keys - 'llm_response' and 'usage' + # --'llm_response' is the output from the model + # --'usage' is a dictionary with the usage stats + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + + # note: the 'gold answer' is the answer we provided above in the hello_world question list + print(f"Gold Answer: {entries['answer']}") + + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + # Step 1 - we will pick a model from the ModelCatalog + + # A few useful methods to discover and display a list of available models... + + # all generative models + llm_models = ModelCatalog().list_generative_models() + + # if you only want to see the local models + llm_local_models = ModelCatalog().list_generative_local_models() + + # to see only the open source models + llm_open_source_models = ModelCatalog().list_open_source_models() + + # we will print out the local models + for i, models in enumerate(llm_local_models): + print("models: ", i, models["model_name"], models["model_family"]) + + # for purposes of demo, try a few selected models from the list + + # each of these pytorch models are ~1b parameters and will run reasonably fast and accurate on CPU + # --per note above, may require separate pip3 install of: torch and transformers + pytorch_generative_models = ["llmware/bling-1b-0.1", "llmware/bling-tiny-llama-v0", "llmware/bling-falcon-1b-0.1"] + + # bling-answer-tool is 1b parameters quantized + # bling-phi-3-gguf is 3.8b parameters quantized + # dragon-yi-6b-gguf is 6b parameters quantized + gguf_generative_models = ["bling-answer-tool", "bling-phi-3-gguf","llmware/dragon-yi-6b-gguf"] + + # by default, we will select a gguf model requiring no additional imports + model_name = gguf_generative_models[0] + + # to swap in a gpt-4 openai model - uncomment these two lines + # model_name = "gpt-4" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + fast_start_prompting(model_name) + diff --git a/solutions/rag/example-4-rag-text-query.py b/solutions/rag/example-4-rag-text-query.py new file mode 100644 index 0000000..5bb911b --- /dev/null +++ b/solutions/rag/example-4-rag-text-query.py @@ -0,0 +1,193 @@ + +""" Fast Start Example #4 - RAG with Text Query + + This example shows a basic RAG recipe using text query combined with LLM prompt. + + We will show two different ways to achieve this basic recipe: + + -- Example 4A - this will integrate Library + Prompt - and is the most scalable general solution + + -- Example 4B - this will illustrate another capability of the Prompt class to add sources "inline" + without necessarily a library in-place. It is another useful tool when you want to be able to quickly + pick up a document and start asking questions to it. + + Note: both of the examples are designed to achieve the same output. + +""" + +import os +import re +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.setup import Setup +from llmware.configs import LLMWareConfig +from llmware.retrieval import Query +from llmware.library import Library + + +def example_4a_contract_analysis_from_library (model_name, verbose=False): + + """ Example #4a: Main general case to run a RAG workflow from a Library """ + + # Load the llmware sample files + print (f"\n > Loading the llmware sample files...") + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path,"Agreements") + + contracts_lib = Library().create_new_library("example4_library") + contracts_lib.add_files(contracts_path) + + # questions that we want to ask each contract + question_list = [{"topic": "executive employment agreement", "llm_query": "What are the names of the two parties?"}, + {"topic": "base salary", "llm_query": "What is the executive's base salary?"}, + {"topic": "governing law", "llm_query": "What is the governing law?"}] + + print (f"\n > Loading model {model_name}...") + + q = Query(contracts_lib) + + # get a list of all of the unique documents in the library + + # doc id list + doc_list = q.list_doc_id() + print("update: document id list - ", doc_list) + + # filename list + fn_list = q.list_doc_fn() + print("update: filename list - ", fn_list) + + prompter = Prompt().load_model(model_name) + + for i, doc_id in enumerate(doc_list): + + print("\nAnalyzing contract: ", str(i+1), doc_id, fn_list[i]) + + print("LLM Responses") + + for question in question_list: + + query_topic = question["topic"] + llm_question = question["llm_query"] + + doc_filter = {"doc_ID": [doc_id]} + query_results = q.text_query_with_document_filter(query_topic,doc_filter,result_count=5,exact_mode=True) + + if verbose: + # this will display the query results from the query above + for j, qr in enumerate(query_results): + print("update: querying document - ", query_topic, j, doc_filter, qr) + + source = prompter.add_source_query_results(query_results) + + # *** this is the call to the llm with the source packaged in the context automatically *** + responses = prompter.prompt_with_source(llm_question, prompt_name="default_with_context", temperature=0.3) + + # unpacking the results from the LLM + for r, response in enumerate(responses): + print("update: llm response - ", llm_question, re.sub("[\n]"," ", response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + + # Save jsonl report to jsonl to /prompt_history folder + print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + prompter.save_state() + + # Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + print("\nCSV output saved at: ", csv_output) + + return 0 + + +def example_4b_contract_analysis_direct_from_prompt(model_name, verbose=False): + + """ Example #4b: Alternative implementation using prompt in-line capabilities without using a library """ + + # Load the llmware sample files + print(f"\n > Loading the llmware sample files...") + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path, "Agreements") + + # questions that we want to ask each contract + question_list = [{"topic": "executive employment agreement", "llm_query": "What are the names of the two parties?"}, + {"topic": "base salary", "llm_query": "What is the executive's base salary?"}, + {"topic": "governing law", "llm_query": "What is the governing law?"}] + + print(f"\n > Loading model {model_name}...") + + prompter = Prompt().load_model(model_name) + + for i, contract in enumerate(os.listdir(contracts_path)): + + # exclude potential mac os created file artifact in the samples folder path + if contract != ".DS_Store": + + print("\nAnalyzing contract: ", str(i + 1), contract) + + print("LLM Responses") + + for question in question_list: + + query_topic = question["topic"] + llm_question = question["llm_query"] + + # introducing "add_source_document" + # this will perform 'inline' parsing, text chunking and query filter on a document + # input is a file folder path, file name, and an optional query filter + # the source is automatically packaged into the prompt object + + source = prompter.add_source_document(contracts_path,contract,query=query_topic) + + if verbose: + print("update: document created source - ", source) + + # calling the LLM with 'source' information from the contract automatically packaged into the prompt + responses = prompter.prompt_with_source(llm_question, prompt_name="default_with_context", + temperature=0.3) + + # unpacking the LLM responses + for r, response in enumerate(responses): + print("update: llm response: ", llm_question, re.sub("[\n]", " ", + response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + + # Save jsonl report to jsonl to /prompt_history folder + print("\nupdate: Prompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(), prompter.prompt_id)) + prompter.save_state() + + # Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + print("\nupdate: CSV output saved at - ", csv_output) + + return 0 + + +if __name__ == "__main__": + + # you can pick any model from the ModelCatalog + # we list a few representative good choices below + + LLMWareConfig().set_active_db("sqlite") + + example_models = ["bling-phi-3-gguf", + "llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/dragon-yi-6b-gguf"] + + # to swap in a gpt-4 openai model - uncomment these two lines and `pip3 install openai` + # model_name = "gpt-4" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # use local cpu model + model_name = example_models[0] + + # two good recipes to address the use case + + # first let's look at the main way of retrieving and analyzing from a library + example_4a_contract_analysis_from_library(model_name) + + # second - uncomment this line, and lets run the "in-line" prompt way + # example_4b_contract_analysis_direct_from_prompt(model_name) + diff --git a/solutions/rag/example-5-rag-semantic-query.py b/solutions/rag/example-5-rag-semantic-query.py new file mode 100644 index 0000000..3dab4f6 --- /dev/null +++ b/solutions/rag/example-5-rag-semantic-query.py @@ -0,0 +1,168 @@ + +""" Fast Start Example #5 - RAG with Semantic Query + + This example illustrates the most common RAG retrieval pattern, which is using a semantic query, e.g., + a natural language query, as the basis for retrieving relevant text chunks, and then using as + the context material in a prompt to ask the same question to a LLM. + + In this example, we will show the following: + + 1. Create library and install embeddings (feel free to skip / substitute a library created in an earlier step). + 2. Ask a general semantic query to the entire library collection. + 3. Select the most relevant results by document. + 4. Loop through all of the documents - packaging the context and asking our questions to the LLM. + + Note: to run this example with the selected embedding pytorch model from the huggingface catalog, + you may need to install additional dependencies: + + `pip3 install transformers` + `pip3 install torch` + + We would recommend any of the following 'no-install' vector db options: + + -- milvus lite: `pip3 install pymilvus` [available starting in llmware>=0.3.0 on Mac/Linux] + -- chromadb: `pip3 install chromadb` + -- lancedb: `pip3 install lancedb` + -- faiss: `pip3 install faiss` + + +""" + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.resources import Status +from llmware.prompts import Prompt +from llmware.configs import LLMWareConfig, MilvusConfig +from importlib import util + +if not util.find_spec("torch") or not util.find_spec("transformers"): + print("\nto run this example, with the selected embedding model, please install transformers and torch, e.g., " + "\n`pip install torch`" + "\n`pip install transformers`") + +if not (util.find_spec("chromadb") or util.find_spec("pymilvus") or util.find_spec("lancedb") or util.find_spec("faiss")): + print("\nto run this example, you will need to pip install the vector db drivers. see comments above.") + + +def semantic_rag (library_name, embedding_model_name, llm_model_name): + + """ Illustrates the use of semantic embedding vectors in a RAG workflow + --self-contained example - will be duplicative with some of the steps taken in other examples """ + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + contracts_path = os.path.join(sample_files_path, "Agreements") + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses all of the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + # -- note: in testing, we have found that the retrieval success is sensitive to the chunking strategy + # -- please keep in mind as you adapt this example with your own documents + library.add_files(input_folder_path=contracts_path, chunk_size=400, max_chunk_size=800, smart_chunking=2) + + # Step 4 - Install the embeddings + print("\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}".format(vector_db, embedding_model)) + + library.install_new_embedding(embedding_model_name=embedding_model_name, vector_db=vector_db, batch_size=200) + + # RAG steps start here ... + + print("\nupdate: Loading model for LLM inference - ", llm_model_name) + + prompter = Prompt().load_model(llm_model_name, temperature=0.0, sample=False) + + query = "what is the executive's base annual salary" + + # key step: run semantic query against the library and get all of the top results + results = Query(library).semantic_query(query, result_count=80, embedding_distance_threshold=1.0) + + # if you want to look at 'results', uncomment the line below + # for i, res in enumerate(results): print("\nupdate: ", i, res["file_source"], res["distance"], res["text"]) + + for i, contract in enumerate(os.listdir(contracts_path)): + + qr = [] + + if contract != ".DS_Store": + + print("\nContract Name: ", i, contract) + + # we will look through the list of semantic query results, and pull the top results for each file + for j, entries in enumerate(results): + + library_fn = entries["file_source"] + if os.sep in library_fn: + # handles difference in windows file formats vs. mac / linux + library_fn = library_fn.split(os.sep)[-1] + + if library_fn == contract: + print("Top Retrieval: ", j, entries["distance"], entries["text"]) + qr.append(entries) + + # we will add the query results to the prompt + source = prompter.add_source_query_results(query_results=qr) + + # run the prompt + response = prompter.prompt_with_source(query, prompt_name="default_with_context") + + # note: prompt_with_resource returns a list of dictionary responses + # -- depending upon the size of the source context, it may call the llm several times + # -- each dict entry represents 1 call to the LLM + + for resp in response: + if "llm_response" in resp: + print("\nupdate: llm answer - ", resp["llm_response"]) + + # start fresh for next document + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + # we will use one of the most popular open source embedding models by jina-ai + # e.g., jinaai/jina-embeddings-v2-base-en + embedding_model = "jina-small-en-v2" + + # Select a 'no install' vector db + + # note: starting with llmware>=0.3.0, we support the new milvus lite - you can ignore or comment out if + # using a different vector db -> note: milvus lite only on mac/linux (not windows) + MilvusConfig().set_config("lite", True) + + # select one of: 'milvus' | 'chromadb' | 'lancedb' | 'faiss' + LLMWareConfig().set_vector_db("chromadb") + + vector_db = "chromadb" + + # pick any name for the library + lib_name = "example_5_library" + + example_models = ["bling-phi-3-gguf", "llmware/bling-1b-0.1", "llmware/dragon-yi-6b-gguf"] + + # use local cpu model + llm_model_name = example_models[0] + + # to swap in a gpt-4 openai model - uncomment these two lines + # llm_model_name = "gpt-4" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + semantic_rag(lib_name, embedding_model, llm_model_name) + + + diff --git a/solutions/rag/example-6-rag-multi-step-query.py b/solutions/rag/example-6-rag-multi-step-query.py new file mode 100644 index 0000000..c8248de --- /dev/null +++ b/solutions/rag/example-6-rag-multi-step-query.py @@ -0,0 +1,116 @@ + +""" Fast Start Example #6 - RAG - Beyond the Basics + + This example builds upon examples #4 and #5 and demonstrates how to layer additional elements to + improve the effectiveness of a RAG workflow over a larger set of documents- + + 1. Apply an initial filter across a batch of documents to identify a subset of documents of interest + 2. Analyze the documents of interest to identify key provisions + 3. Use fact-checking and post-processing to validate the accuracy of the LLM response + 4. Write the output to JSON and CSV files for follow-up review and/or the next step in the workflow. + + For this example, we also recommend using a more sophisticated DRAGON model in GGUF format, which enables us + to run 6-7B parameter models locally. + +""" + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + + +def msa_processing(library_name, llm_model_name): + + """ In this example, we will use the 'AgreementsLarge' sample files which consists of ~80 contracts. We + need to quickly identify the 'master service agreements' as we only want to analyze those contracts. """ + + local_path = Setup().load_sample_files() + agreements_path = os.path.join(local_path, "AgreementsLarge") + + # create a library with all of the Agreements (~80 contracts) + print(f"\nStarting: Parsing 'AgreementsLarge' Folder") + msa_lib = Library().create_new_library(library_name) + msa_lib.add_files(agreements_path) + + # find the "master service agreements" (MSA) - we know that 'master services agreement' will always + # be on the first page of the agreement, so we can use that as a good proxy for automatically filtering + # to our target set of documents + + print(f"\nCompleted Parsing - now, let's look for the 'master service agreements', e.g., 'msa'") + + q = Query(msa_lib) + query = '"master services agreement"' + results = q.text_search_by_page(query, page_num=1, results_only=False) + + # results_only = False will return a dictionary with 4 keys: {"query", "results", "doc_ID", "file_source"} + msa_docs = results["file_source"] + msa_doc_ids = results["doc_ID"] + + # load prompt/llm locally + prompter = Prompt().load_model(llm_model_name) + + print("update: identified the following msa doc id: ", msa_doc_ids) + + # analyze each MSA - "query" & "llm prompt" + for i, doc_id in enumerate(msa_doc_ids): + + print("\n") + docs = msa_docs[i] + if os.sep in docs: + # handles difference in windows file formats vs. Mac/Linux + docs = docs.split(os.sep)[-1] + + print (i+1, "Reviewing MSA - ", doc_id, docs) + + # look for the termination provisions in each document + doc_filter = {"doc_ID": [doc_id]} + termination_provisions = q.text_query_with_document_filter("termination", doc_filter) + + # package the provisions as a source to a prompt + sources = prompter.add_source_query_results(termination_provisions) + + # if you want to see more details about how the sources are packaged: uncomment this line- + # print("update: sources - ", sources) + + # call the LLM and ask our question + response = prompter.prompt_with_source("What is the notice for termination for convenience?") + + # post processing fact checking + stats = prompter.evidence_comparison_stats(response) + ev_source = prompter.evidence_check_sources(response) + + for j, resp in enumerate(response): + print("update: llm response - ", resp) + print("update: compare with evidence- ", stats[j]["comparison_stats"]) + print("update: sources - ", ev_source[j]["source_review"]) + + prompter.clear_source_materials() + + # Save jsonl report with full transaction history to /prompt_history folder + print("\nupdate: Prompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + + prompter.save_state() + + # Generate CSV report for easy Human review in Excel + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + + print("\nupdate: CSV output for human review - ", csv_output) + + return 0 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + # this is part of the DRAGON model series - RAG-fine-tuned fact-based Q&A model + llm = "bling-phi-3-gguf" + + # feel free to also try: "dragon-yi-answer-tool" as a good substitute option + + m = msa_processing("example6_library", llm) + diff --git a/solutions/rag/example-7-function-calls.py b/solutions/rag/example-7-function-calls.py new file mode 100644 index 0000000..c0accd9 --- /dev/null +++ b/solutions/rag/example-7-function-calls.py @@ -0,0 +1,204 @@ + +""" Fast Start Example #7 - Model Function Calls + + This example is derived from the example in: examples/SLIM-Agents/using_function_calls.py + + Objective: start moving beyond basic prompting and question-answering, and start using LLMs for + "function calls" with structured programmatic outputs + + Generally, function-calling is a specialized capability of frontier language models, such as OpenAI GPT4. + + We have adapted this concept to small language models through SLIMs (Structured Language Instruction Models), + which are 'single function' models fine-tuned to accept three main inputs to construct a prompt and generate a structured output. + + As of June 2024, there are 18 distinct SLIM function calling models with many more on the way, for most common + extraction, classification, and summarization tasks. + + All SLIM models have a common prompting structure + + Inputs: + -- text passage - this is the core passage or piece of text that you would like the model to assess + -- function - classify, extract, generate - this is handled by default by the model class, so usually does + not need to be explicitly declared - but is an option for SLIMs that support more than one function + -- params - depends upon the model, used to configure/guide the behavior of the function call - optional for + some SLIMs + + Outputs: + -- structured python output, generally either a dictionary or list + + Main objectives: + -- enable function calling with small, locally-running models, + -- simplify prompts by defining specific functions and fine-tuning the model to respond accordingly + without 'prompt magic' + -- standardized outputs that can be handled programmatically as part of a multi-step workflow. + """ + + +from llmware.models import ModelCatalog + + +def discover_slim_models(): + + """ Discover a list of SLIM tools in the Model Catalog. + + -- SLIMs are available in both traditional Pytorch and quantized GGUF packages. + -- Generally, we train/fine-tune in Pytorch and then package in 4-bit quantized GGUF for inference. + -- By default, we designate the GGUF versions with 'tool' or 'gguf' in their names. + -- GGUF versions are generally faster to load, faster for inference and use less memory in most environments.""" + + tools = ModelCatalog().list_llm_tools() + tool_map = ModelCatalog().get_llm_fx_mapping() + + print("\nList of SLIM model tools (GGUF) in the ModelCatalog\n") + + for i, tool in enumerate(tools): + model_card = ModelCatalog().lookup_model_card(tool_map[tool]) + print(f"{i} - tool: {tool} - " + f"model_name: {model_card['model_name']} - " + f"model_family: {model_card['model_family']}") + + return 0 + + +def hello_world_slim(): + + """ SLIM models can be identified in the ModelCatalog like any llmware model. Instead of using + inference method, SLIM models are used with the function_call method that prepares a special prompt + instruction, and takes optional parameters. + + This example shows a series of function calls with different SLIM models. + + Please note that the first time the models will be pulled from the llmware Huggingface repository, and will + take a couple of minutes. Future calls will be much faster once cached in memory locally. """ + + print("\nExecuting Function Call Inferences with SLIMs\n") + + # Sentiment Analysis + + passage1 = ("This is one of the best quarters we can remember for the industrial sector " + "with significant growth across the board in new order volume, as well as price " + "increases in excess of inflation. We continue to see very strong demand, especially " + "in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would " + "be accumulating more stock on any dips.") + + # here are the two key lines of code + model = ModelCatalog().load_model("slim-sentiment-tool") + response = model.function_call(passage1) + + print("sentiment response: ", response['llm_response']) + + # Named Entity Recognition + + passage2 = "Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s." + + model = ModelCatalog().load_model("slim-ner-tool") + response = model.function_call(passage2) + + print("ner response: ", response['llm_response']) + + # Extract anything with Slim-extract + + passage3 = ("Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee.") + + model = ModelCatalog().load_model("slim-extract-tool") + response = model.function_call(passage3, function="extract", params=["revenue growth"]) + + print("extract response: ", response['llm_response']) + + # Generate questions with Slim-Q-Gen + + model = ModelCatalog().load_model("slim-q-gen-tiny-tool", temperature=0.2, sample=True) + # supported params - "question", "multiple choice", "boolean" + response = model.function_call(passage3, params=['multiple choice']) + + print("question generation response: ", response['llm_response']) + + # Generate topic + + model = ModelCatalog().load_model("slim-topics-tool") + response = model.function_call(passage3) + + print("topics response: ", response['llm_response']) + + # Generate headline summary with slim-xsum + model = ModelCatalog().load_model("slim-xsum-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("xsum response: ", response['llm_response']) + + # Generate boolean with optional '(explain)` in parameter + model = ModelCatalog().load_model("slim-boolean-tool") + response = model.function_call(passage3, params=["Did Adobe revenue increase? (explain)"]) + + print("boolean response: ", response['llm_response']) + + # Generate tags + model = ModelCatalog().load_model("slim-tags-tool", temperature=0.0, sample=False) + response = model.function_call(passage3) + + print("tags response: ", response['llm_response']) + + return 0 + + +def using_logits_and_integrating_into_process(): + + """ This example shows two key elements of function calling SLIM models - + + 1. Using Logit Information to indicate confidence levels, especially for classifications. + 2. Using the structured dictionary generated for programmatic handling in a larger process. + + """ + + print("\nExample: using logits and integrating into process\n") + + text_passage = ("On balance, this was an average result, with earnings in line with expectations and " + "no big surprises to either the positive or the negative.") + + # two key lines (load_model + execute function_call) + additional logit_analysis step + sentiment_model = ModelCatalog().load_model("slim-sentiment-tool", get_logits=True) + response = sentiment_model.function_call(text_passage) + analysis = ModelCatalog().logit_analysis(response,sentiment_model.model_card, sentiment_model.hf_tokenizer_name) + + print("sentiment response: ", response['llm_response']) + + print("\nAnalyzing response") + for keys, values in analysis.items(): + print(f"{keys} - {values}") + + # two key attributes of the sentiment output dictionary + sentiment_value = response["llm_response"]["sentiment"] + confidence_level = analysis["confidence_score"] + + # use the sentiment classification as a 'if...then' decision point in a process + if "positive" in sentiment_value: + print("sentiment is positive .... will take 'positive' analysis path ...", sentiment_value) + else: + print("sentiment is negative .... will take 'negative' analysis path ...", sentiment_value) + + if "positive" in sentiment_value and confidence_level > 0.8: + print("sentiment is positive with high confidence ... ", sentiment_value, confidence_level) + + return 0 + + +if __name__ == "__main__": + + # discovering slim models in the llmware catalog + discover_slim_models() + + # running function call inferences + hello_world_slim() + + # doing interesting stuff with the output + using_logits_and_integrating_into_process() + + diff --git a/solutions/rag/example-8-agents.py b/solutions/rag/example-8-agents.py new file mode 100644 index 0000000..ae00f0b --- /dev/null +++ b/solutions/rag/example-8-agents.py @@ -0,0 +1,86 @@ + +""" Fast Start Example #8 - Agents + + This example shows how to build locally-running Agents deploying multiple small specialized function-calling + models as tools with an integrated work management, process and journaling capability using the LLMfx class. + + This example shows a workflow receiving a customer transcript, and having an agent run through a series of + analytical and classification steps. + + 1. Create an agent using the LLMfx class. + 2. Load multiple specialized tools for the agent. + 3. Execute a series of function-calls. + 4. Generate a consolidated automatic dictionary report. + +""" + +from llmware.agents import LLMfx + + +def create_multistep_report(customer_transcript): + + """ Creating a multi-step, multi-model agent workflow """ + + # create an agent using LLMfx class + agent = LLMfx() + + agent.load_work(customer_transcript) + + # load tools individually + agent.load_tool("sentiment") + agent.load_tool("ner") + + # load multiple tools + agent.load_tool_list(["emotions", "topics", "intent", "tags", "ratings", "answer"]) + + # start deploying tools and running various analytics + + # first conduct three 'soft skills' initial assessment using 3 different models + agent.sentiment() + agent.emotions() + agent.intent() + + # alternative way to execute a tool, passing the tool name as a string + agent.exec_function_call("ratings") + + # call multiple tools concurrently + agent.exec_multitool_function_call(["ner","topics","tags"]) + + # the 'answer' tool is a quantized question-answering model - ask an 'inline' question + # the optional 'key' assigns the output to a dictionary key for easy consolidation + agent.answer("What is a short summary?",key="summary") + + # prompting tool to ask a quick question as part of the analytics + response = agent.answer("What is the customer's account number and user name?", key="customer_info") + + # you can 'unload_tool' to release it from memory + agent.unload_tool("ner") + agent.unload_tool("topics") + + # at end of processing, show the report that was automatically aggregated by key + report = agent.show_report() + + # displays a summary of the activity in the process + activity_summary = agent.activity_summary() + + # list of the responses gathered + for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + + output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + + return output + + +if __name__ == "__main__": + + # sample customer transcript + + customer_transcript = "My name is Michael Jones, and I am a long-time customer. " \ + "The Mixco product is not working currently, and it is having a negative impact " \ + "on my business, as we can not deliver our products while it is down. " \ + "This is the fourth time that I have called. My account number is 93203, and " \ + "my user name is mjones. Our company is based in Tampa, Florida." + + output = create_multistep_report(customer_transcript) + diff --git a/solutions/rag/example-9-function-calls-with-web-services.py b/solutions/rag/example-9-function-calls-with-web-services.py new file mode 100644 index 0000000..1a6459e --- /dev/null +++ b/solutions/rag/example-9-function-calls-with-web-services.py @@ -0,0 +1,207 @@ + +""" Fast Start #9 - Function Calls with Web Services + + This example illustrates one of the most exciting combinations in LLM-based applications, specifically + combining function calls with external web services to drive more complex automation patterns. + + This Fast Start example is derived from the original example at: /examples/Use_Cases/web_services_slim_fx.py + + Models + 1. slim-extract-tool + 2. slim-summary-tool + 3. bling-stablelm-3b-tool + + Web Services + 1. Yfinance - stock ticker + 2. Wikipedia - company background information + + The example shows how to extract keys from one source that can then be used as a lookup in a web service to + supplement the original source materials, and provide a secondary source, which can then also be prompted and + used to extract, analyze and summarize key information. + + NOTE: to run this example, please install yfinance library, e.g., 'pip3 install yfinance' + + """ + + +from llmware.web_services import YFinance +from llmware.models import ModelCatalog +from llmware.parsers import WikiParser + +from importlib import util +if not util.find_spec("yfinance"): + print("\nto run this example, you need to install yfinance first, e.g., pip3 install yfinance") + +# our input - financial news article + +text=("BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its " + "third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter " + "of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* " + "at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis " + "NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis " + "Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin " + "increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges " + "Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring " + "charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these " + "charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to " + "drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by " + "the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and " + "work with our wholesale partners to elevate and grow the marketplace.") + + +def research_example1(): + + """ End-to-end example generating 30 output key:value pairs """ + + # load three models in this example + + model = ModelCatalog().load_model("slim-extract-tool", temperature=0.0, sample=False) + model2 = ModelCatalog().load_model("slim-summary-tool", sample=False,temperature=0.0,max_output=200) + model3 = ModelCatalog().load_model("bling-stablelm-3b-tool", sample=False, temperature=0.0) + + research_summary = {} + + # extract information from the source materials + + extract_keys = ["stock ticker", "company name", + "total revenues", "restructuring charges", + "digital growth", "ceo comment", "quarter end date"] + + print("\nStep 1 - extract information from source text\n") + + for keys in extract_keys: + response = model.function_call(text,params=[keys]) + dict_keys = keys.replace(" ", "_") + print(f"update: extracting - {keys} - {response['llm_response']}") + if dict_keys in response["llm_response"]: + value = response["llm_response"][dict_keys][0] + research_summary.update({dict_keys: value}) + else: + print("could not find look up key successfully - ", response["llm_response"]) + + # secondary lookups using extracted information + + print("\nStep 2 - use extracted stock ticker in web service lookup to YFinance\n") + + if "stock_ticker" in research_summary: + ticker = research_summary["stock_ticker"] + # a little kludge related to yfinance api + ticker_core = ticker.split(":")[-1] + + yf = YFinance().get_stock_summary(ticker=ticker_core) + print("yahoo finance stock info: ", yf) + + research_summary.update({"current_stock_price": yf["currentPrice"]}) + research_summary.update({"high_ltm": yf["fiftyTwoWeekHigh"]}) + research_summary.update({"low_ltm": yf["fiftyTwoWeekLow"]}) + research_summary.update({"trailing_pe": yf["trailingPE"]}) + research_summary.update({"forward_pe": yf["forwardPE"]}) + research_summary.update({"volume": yf["volume"]}) + + yf2 = YFinance().get_financial_summary(ticker=ticker_core) + print("yahoo finance fin info - ", yf2) + research_summary.update({"market_cap": yf2["marketCap"]}) + research_summary.update({"price_to_sales": yf2["priceToSalesTrailing12Months"]}) + research_summary.update({"revenue_growth": yf2["revenueGrowth"]}) + research_summary.update({"ebitda": yf2["ebitda"]}) + research_summary.update({"gross_margin": yf2["grossMargins"]}) + research_summary.update({"currency": yf2["currency"]}) + + yf3 = YFinance().get_company_summary(ticker=ticker_core) + print("yahoo finance company info - ", yf3) + research_summary.update({"sector": yf3["sector"]}) + research_summary.update({"website": yf3["website"]}) + research_summary.update({"industry": yf3["industry"]}) + research_summary.update({"employees": yf3["fullTimeEmployees"]}) + + execs = [] + if "companyOfficers" in yf3: + for entries in yf3["companyOfficers"]: + if "totalPay" in entries: + pay = entries["totalPay"] + else: + pay = "pay-NA" + + if "age" in entries: + age = entries["age"] + else: + age = "age-NA" + + execs.append((entries["name"], entries["title"], age, pay)) + research_summary.update({"officers": execs}) + + print("\nStep 3 - use extracted company name to lookup in Wikipedia web service - and add background data\n") + + if "company_name" in research_summary: + + company_name = research_summary["company_name"] + output = WikiParser().add_wiki_topic(company_name, target_results=1) + + # get company summary + company_overview = "" + for i, blocks in enumerate(output["blocks"]): + if i < 3: + company_overview += blocks["text"] + + # call summary model to summarize + print("-- calling summary model to summarize the first part of the Wikipedia article") + summary = model2.function_call(company_overview, params=["company history (5)"]) + print("-- slim-summary - summary (5 points): ", summary) + + research_summary.update({"summary": summary["llm_response"]}) + + # get founding date + print("\n-- calling extract model to get key piece of information from the Wikipedia article - company founding date") + response = model.function_call(company_overview, params=["founding date"]) + print("-- slim-extract - founding date: ", response) + + research_summary.update({"founding_date": response["llm_response"]["founding_date"][0]}) + + print("\n-- calling extract model to get a short company business") + response = model.function_call(company_overview, params=["company description"]) + print("-- slim-extract - response: ", response) + research_summary.update({"company_description": response["llm_response"]["company_description"][0]}) + + # ask other questions directly + print("\n-- asking a question directly to the Wikipedia article about the business") + response = model3.inference("What is an overview of company's business?", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"business_overview": response["llm_response"] }) + + print("\n-- asking a question about the origin of the company's name") + response = model3.inference("What is the origin of the company's name?", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"origin_of_name": response["llm_response"]}) + + print("\n-- asking a question about the company's products") + response = model3.inference("What are the product names", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"products": response["llm_response"]}) + + print("\n\nStep 4 - Completed Research - Summary Output\n") + print("research summary: ", research_summary) + + item_counter = 1 + + for keys, values in research_summary.items(): + if isinstance(values, str): + + values = values.replace("\n", "") + values = values.replace("\r", "") + values = values.replace("\t", "") + + print(f"\t\t -- {item_counter} - \t - {keys.ljust(25)} - {str(values).ljust(40)}") + item_counter += 1 + + return research_summary + + +if __name__ == "__main__": + + research_example1() + + + + + + diff --git a/solutions/slim_agents/README 2.md b/solutions/slim_agents/README 2.md new file mode 100644 index 0000000..646642a --- /dev/null +++ b/solutions/slim_agents/README 2.md @@ -0,0 +1,80 @@ + + +Fast Start: Building Agent workflows with small language models with llmware +=============== + +**Welcome to llmware!** + +Set up + +`pip3 install llmware` or `pip3 install 'llmware[full]'` or, if you prefer clone the github repo locally, e.g., `git clone git@github.com:llmware-ai/llmware.git`. If you clone the repo, then we would recommend that you run the `welcome_to_llmware.sh` or `welcome_to_llmware_windows.sh` scripts to install all of the dependencies. + +Platforms: +- Mac M1/M2/M3, Windows, Linux (Ubuntu 20 or Ubuntu 22 preferred) +- RAM: 16 GB minimum (32 GB recommended) +- Python 3.9, 3.10, 3.11, 3.12 + +**What is an Agent in llmware?** + +There are a lot of different industry definitions of an Agent or an agent-based process. Our implementation is very specific in focusing on building multi-step, multi-model workflows that can be instantiated and run entirely locally or in a self-hosted manner. We use small specialized models that are "tools" that can be easily stacked together as part of building a more complex pipeline consisting of multiple calls to LLMs, along with other processing logic. + +In short, we see agents as the way to evolve beyond simple chatbots, and start using LLMs to unlock enterprise process automation, and integrating LLMs safely, securely and cost-effectively into private enterprise workflows. + +Each of these examples below will walk you through the basics of how to start using models in llmware, and then how to start composing more complex applications by combining different combinations of models and related tools. + +There are 15 examples, designed to be used step-by-step, but each is self-contained, so you can feel free to jump into any of the examples, in any order, that you prefer. + +Each example has been designed to be "copy-paste" and RUN with lots of helpful comments and explanations embedded in the code samples. + +Examples: + +1. **Start here** - start downloading and running question-answering and function-calling models in minutes. + +2. **llmware_sampler_bling_dragon** - get started with BLING and DRAGON models for high-quality, fact-based inferencing. + +3. **using-slim-extract** - start using function-calling small specialized models for extracting information from documents. + +4. **using-slim-summary** - start using function-calling small specialized models for summarizing information. + +5. **agent-llmfx** - build your first agent process and run it all locally. + +6. **agent-multistep-process** - a second example of a multi-step agent process to analyze, classify and extract information from a complex document. + +7. **using-whisper** - voice transcription to text in minutes, running locally with whisper-cpp. + +8. **using-phi-3-function-calls** - using phi3-mini for various function call processes. + +9. **summarize_document** - summarizing a larger document in multiple chunks of the document and then assembling. + +10. **semantic similarity ranking** - using a semantic reranker to filter and build relevant text chunks from larger documents. + +11. **gguf_streaming** - use the stream interface to stream text for larger generations. + +12. **web_services** - integrate web services to build a complex research report, combined with three distinct function calling models. + +13. **text-2-sql** - convert natural language queries into SQL and extract information from structured databases. + +14. **rag-instruct-benchark-tester** - script for building rag benchmark performance tests. + +15. **using-rag-benchmark-scores** - how to access and filter models by ranking accuracy on the benchmark test. + + +After completing these 15 examples, you should have a good foundation and set of recipes to start +exploring the other 100+ examples in the /examples folder, and build more sophisticated +LLM-based applications. + +**Models** + - All of these examples are optimized for using local CPU-based models, primarily BLING, DRAGON and SLIM models. + - If you want to substitute for any other model in the catalog, it is generally as easy as + switching the model_name. If the model requires API keys, we show in the examples how to pass those keys as an + environment variable. + +**Local Private** + - All of the processing will take place locally on your laptop. + +*This is an ongoing initiative to provide easy-to-get-started tutorials - we welcome and encourage feedback, as well +as contributions with examples and other tips for helping others on their LLM application journeys!* + +**Let's get started!** + + diff --git a/solutions/slim_agents/README.md b/solutions/slim_agents/README.md new file mode 100644 index 0000000..685c073 --- /dev/null +++ b/solutions/slim_agents/README.md @@ -0,0 +1,87 @@ + 🚀 Start Building Multi-Model Agents Locally on a Laptop 🚀 +=============== + +**What is a SLIM?** + +**SLIMs** are **S**tructured **L**anguage **I**nstruction **M**odels, which are small, specialized 1-3B parameter LLMs, +finetuned to generate structured outputs (Python dictionaries and lists, JSON and SQL) that can be handled programmatically, and +stacked together in multi-step, multi-model Agent workflows - all running on a local CPU. + +**New SLIMS Just released** - check out slim-extract, slim-summarize, slim-xsum, slim-sa-ner, slim-boolean and slim-tags-3b + +**Check out the new examples below marked with ⭐** +🔥🔥🔥 Web Services & Function Calls ([code](web_services_slim_fx.py)) 🔥🔥🔥 + +**Check out the Intro videos** +[SLIM Intro Video](https://www.youtube.com/watch?v=cQfdaTcmBpY) + +There are 16 SLIM models, each delivered in two packages - a Pytorch/Huggingface FP16 model, and a +quantized "tool" designed for fast inference on a CPU, using LLMWare's embedded GGUF inference engine. In most cases, +we would recommend that you start with the "tools" version of the models. + +**Getting Started** + +We have several ready-to-run examples in this repository: + +| Example | Detail | +|-----------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------| +| 1. Getting Started with SLIM Models ([code](slims-getting-started.py) / [video](https://www.youtube.com/watch?v=aWZFrTDmMPc&t=196s)) | Install the models and run hello world tests to see the models in action. | +| 2. Getting Started with Function-Calling Agent ([code](agent-llmfx-getting-started.py) / [video](https://www.youtube.com/watch?v=cQfdaTcmBpY)) | Generate a Structured Report with LLMfx | +| 3. Multi-step Complex Analysis with Agent ([code](agent-multistep-analysis.py) / [video](https://www.youtube.com/watch?v=y4WvwHqRR60)) | Delivering Complex Research Analysis with SLIM Agents | | +| 4. Document Clustering ([code](document-clustering.py)) | Multi-faceted automated document analysis with Topics, Tags and NER | +| 5. Two-Step NER Retrieval ([code](ner-retrieval.py)) | Using NER to extract name, and then using as basis for retrieval. | | +| 6. Using Sentiment Analysis ([code](sentiment-analysis.py)) | Using sentiment analysis on earnings transcripts and a 'if...then' condition | +| 7. Text2SQL - Intro ([code](text2sql-getting-started-1.py)) | Getting Started with SLIM-SQL-TOOL and Basic Text2SQL Inference | | +| 8. Text2SQL - E2E ([code](text2sql-end-to-end-2.py)) | End-to-End Natural Langugage Query to SQL DB Query | | +| 9. Text2SQL - MultiStep ([code](text2sql-multistep-example-3.py)) | Extract a customer name using NER and use in a Text2SQL query | +| 10. ⭐ Web Services & Function Calls ([code](web_services_slim_fx.py)) | Generate 30 key financial analysis with SLIM function calls and web services | +| 11. ⭐ Yes-No Questions with Explanations ([code](using_slim_boolean_model.py)) | Analyze earnings releases with SLIM Boolean | +| 12. ⭐ Extracting Revenue Growth ([code](using_slim_extract_model.py)) | Extract revenue growth from earnings releases with SLIM Extract | +| 13. ⭐ Summary as a Function Call ([code](using_slim_summary.py)) | Simple Summarization as a Function Call with List Length Parameters | +| 14. ⭐ Handling Not Found Extracts ([code](not_found_extract_with_lookup.py)) | Multi-step Lookup strategy and handling not-found answers | +| 15. ⭐ Extract + Lookup ([code](custom_extract_and_lookup.py)) | Extract Named Entity information and use for lookups with SLIM Extract | +| 16. ⭐ Headline/Title as XSUM Function Call ([code](using_slim_xsum.py)) | eXtreme Summarization (XSUM) with SLIM XSUM | + +For information on all of the SLIM models, check out [LLMWare SLIM Model Collection](https://www.huggingface.co/llmware/). + +**Models List** +If you would like more information about any of the SLIM models, please check out their model card: + +- extract - extract custom keys - [slim-extract](https://www.huggingface.co/llmware/slim-extract) & [slim-extract-tool](https://www.huggingface.co/llmware/slim-extract-tool) +- summary - summarize function call - [slim-summary](https://www.huggingface.co/llmware/slim-summary) & [slim-summary-tool](https://www.huggingface.co/llmware/slim-summary-tool) +- xsum - title/headline function call - [slim-xsum](https://www.huggingface.co/llmware/slim-xsum) & [slim-xsum-tool](https://www.huggingface.co/llmware/slim-xsum-tool) +- ner - extract named entities - [slim-ner](https://www.huggingface.co/llmware/slim-ner) & [slim-ner-tool](https://www.huggingface.co/llmware/slim-ner-tool) +- sentiment - evaluate sentiment - [slim-sentiment](https://www.huggingface.co/slim-sentiment) & [slim-sentiment-tool](https://www.huggingface.co/llmware/slim-sentiment-tool) +- topics - generate topic - [slim-topics](https://www.huggingface.co/slim-topics) & [slim-topics-tool](https://www.huggingface.co/llmware/slim-topics-tool) +- sa-ner - combo model (sentiment + named entities) - [slim-sa-ner](https://www.huggingface.co/slim-sa-ner) & [slim-sa-ner-tool](https://www.huggingface.co/llmware/slim-sa-ner-tool) +- boolean - provides a yes/no output with explanation - [slim-boolean](https://www.huggingface.co/slim-boolean) & [slim-boolean-tool](https://www.huggingface.com/llmware/slim-boolean-tool) +- ratings - apply 1 (low) - 5 (high) rating - [slim-ratings](https://www.huggingface.co/slim-ratings) & [slim-ratings-tool](https://www.huggingface.co/llmware/slim-ratings-tool) +- emotions - assess emotions - [slim-emotions](https://www.huggingface.co/slim-emotions) & [slim-emotions-tool](https://www.huggingface.co/llmware/slim-emotions-tool) +- tags - auto-generate list of tags - [slim-tags](https://www.huggingface.co/slim-tags) & [slim-tags-tool](https://www.huggingface.co/llmware/slim-tags-tool) +- tags-3b - enhanced auto-generation tagging model - [slim-tags-3b](https://www.huggingface.com/slim-tags-3b) & [slim-tags-3b-tool](https://www.huggingface.co/llmware/slim-tags-3b-tool) +- intent - identify intent - [slim-intent](https://www.huggingface.co/slim-intent) & [slim-intent-tool](https://www.huggingface.co/llmware/slim-intent-tool) +- category - high-level category - [slim-category](https://www.huggingface.co/slim-category) & [slim-category-tool](https://wwww.huggingface.co/llmware/slim-category-tool) +- nli - assess if evidence supports conclusion - [slim-nli](https://www.huggingface.co/slim-nli) & [slim-nli-tool](https://www.huggingface.co/llmware/slim-nli-tool) +- sql - convert text into sql - [slim-sql](https://www.huggingface.co/slim-sql) & [slim-sql-tool](https://www.huggingface.co/llmware/slim-sql-tool) + +You may also want to check out these quantized 'answer' tools, which work well in conjunction with SLIMs for question-answer and summarization: +- bling-stablelm-3b-tool - 3b quantized RAG model - [bling-stablelm-3b-gguf](https://www.huggingface.co/llmware/bling-stablelm-3b-gguf) +- bling-answer-tool - 1b quantized RAG model - [bling-answer-tool](https://www.huggingface.co/llmware/bling-answer-tool) +- dragon-yi-answer-tool - 6b quantized RAG model - [dragon-yi-answer-tool](https://www.huggingface.co/llmware/dragon-yi-answer-tool) +- dragon-mistral-answer-tool - 7b quantized RAG model - [dragon-mistral-answer-tool](https://www.huggingface.co/llmware/dragon-mistral-answer-tool) +- dragon-llama-answer-tool - 7b quantized RAG model - [dragon-llama-answer-tool](https://www.huggingface.co/llmware/dragon-llama-answer-tool) + + +**Set up** +No special setup for SLIMs is required other than to install llmware >=0.2.6, e.g., `pip3 install llmware`. + +**Platforms:** +- Mac M1, Mac x86, Windows, Linux (Ubuntu 22 preferred, supported on Ubuntu 20 +) +- RAM: 16 GB minimum +- Python 3.9, 3.10, 3.11 (note: not supported on 3.12 yet) +- llmware >= 0.2.6 version + + +### **Let's get started! 🚀** + + diff --git a/solutions/slim_agents/agent-llmfx-getting-started.py b/solutions/slim_agents/agent-llmfx-getting-started.py new file mode 100644 index 0000000..c526448 --- /dev/null +++ b/solutions/slim_agents/agent-llmfx-getting-started.py @@ -0,0 +1,82 @@ + +""" Using SLIM tools as part of an agent workflow - introducing LLMfx class - this example shows how to: + + 1. Create an agent using the LLMfx class. + 2. Load multiple specialized tools for the agent. + 3. Execute a series of function-calls. + 4. Generate a consolidated automatic dictionary report. + +""" + + +from llmware.models import ModelCatalog +from llmware.agents import LLMfx + + +def create_multistep_report(customer_transcript): + + """ Creating a multi-step, multi-model agent workflow """ + + # create an agent using LLMfx class + agent = LLMfx() + + agent.load_work(customer_transcript) + + # load tools individually + agent.load_tool("sentiment") + agent.load_tool("ner") + + # load multiple tools + agent.load_tool_list(["emotions", "topics", "intent", "tags", "ratings", "answer"]) + + # start deploying tools and running various analytics + + # first conduct three 'soft skills' initial assessment using 3 different models + agent.sentiment() + agent.emotions() + agent.intent() + + # alternative way to execute a tool, passing the tool name as a string + agent.exec_function_call("ratings") + + # call multiple tools concurrently + agent.exec_multitool_function_call(["ner","topics","tags"]) + + # the 'answer' tool is a quantized question-answering model - ask an 'inline' question + # the optional 'key' assigns the output to a dictionary key for easy consolidation + agent.answer("What is a short summary?",key="summary") + + # prompting tool to ask a quick question as part of the analytics + response = agent.answer("What is the customer's account number and user name?", key="customer_info") + + # you can 'unload_tool' to release it from memory + agent.unload_tool("ner") + agent.unload_tool("topics") + + # at end of processing, show the report that was automatically aggregated by key + report = agent.show_report() + + # displays a summary of the activity in the process + activity_summary = agent.activity_summary() + + # list of the responses gathered + for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + + output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + + return output + + +if __name__ == "__main__": + + # sample customer transcript + + customer_transcript = "My name is Michael Jones, and I am a long-time customer. " \ + "The Mixco product is not working currently, and it is having a negative impact " \ + "on my business, as we can not deliver our products while it is down. " \ + "This is the fourth time that I have called. My account number is 93203, and " \ + "my user name is mjones. Our company is based in Tampa, Florida." + + output = create_multistep_report(customer_transcript) + diff --git a/solutions/slim_agents/agent-multistep-analysis.py b/solutions/slim_agents/agent-multistep-analysis.py new file mode 100644 index 0000000..41a3cab --- /dev/null +++ b/solutions/slim_agents/agent-multistep-analysis.py @@ -0,0 +1,102 @@ + +""" This example shows a complex multi-part research analysis. In this example, we will: + + 1. Build a "research" library. + 2. Query the research library to identify topics of interest. + 3. Create an agent with several analytical tools: sentiment, emotions, topic, entities analysis + 4. Pass the results of our query to the agent to conduct multifaceted analysis. + 5. Apply a top-level filter ('sentiment') on the results from the query + 6. For any of the passages with negative sentiment, we will run a follow-up set of analyses. + 7. Finally, we will assemble the follow-up analysis into a list of detailed reports. +""" + +import os +import shutil + +from llmware.agents import LLMfx +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig +from llmware.setup import Setup + + +def multistep_analysis(): + + """ In this example, our objective is to research Microsoft history and rivalry in the 1980s with IBM. """ + + # step 1 - assemble source documents and create library + + print("update: Starting example - agent-multistep-analysis") + + # note: lines 38-49 attempt to automatically pull sample document into local path + # depending upon permissions in your environment, you may need to set up directly + # if you pull down the samples files with Setup().load_sample_files(), in the Books folder, + # you will find the source: "Bill-Gates-Biography.pdf" + # if you have pulled sample documents in the past, then to update to latest: set over_write=True + + print("update: Loading sample files") + + sample_files_path = Setup().load_sample_files(over_write=False) + bill_gates_bio = "Bill-Gates-Biography.pdf" + path_to_bill_gates_bio = os.path.join(sample_files_path, "Books", bill_gates_bio) + + microsoft_folder = os.path.join(LLMWareConfig().get_tmp_path(), "example_microsoft") + + print("update: attempting to create source input folder at path: ", microsoft_folder) + + if not os.path.exists(microsoft_folder): + os.mkdir(microsoft_folder) + os.chmod(microsoft_folder, 0o777) + shutil.copy(path_to_bill_gates_bio,os.path.join(microsoft_folder, bill_gates_bio)) + + # create library + print("update: creating library and parsing source document") + + LLMWareConfig().set_active_db("sqlite") + my_lib = Library().create_new_library("microsoft_history_0210_1") + my_lib.add_files(microsoft_folder) + + # run our first query - "ibm" + query = "ibm" + search_results = Query(my_lib).text_query(query) + print(f"update: executing query to filter to key passages - {query} - results found - {len(search_results)}") + + # create an agent and load several tools that we will be using + agent = LLMfx() + agent.load_tool_list(["sentiment", "emotions", "topic", "tags", "ner", "answer"]) + + # load the search results into the agent's work queue + agent.load_work(search_results) + + while True: + + agent.sentiment() + + if not agent.increment_work_iteration(): + break + + # analyze sections where the sentiment on ibm was negative + follow_up_list = agent.follow_up_list(key="sentiment", value="negative") + + for job_index in follow_up_list: + + # follow-up 'deep dive' on selected text that references ibm negatively + agent.set_work_iteration(job_index) + agent.exec_multitool_function_call(["tags", "emotions", "topics", "ner"]) + agent.answer("What is a brief summary?", key="summary") + + my_report = agent.show_report(follow_up_list) + + activity_summary = agent.activity_summary() + + for entries in my_report: + print("my report entries: ", entries) + + return my_report + + +if __name__ == "__main__": + + multistep_analysis() + + diff --git a/solutions/slim_agents/agents-1-start_here.py b/solutions/slim_agents/agents-1-start_here.py new file mode 100644 index 0000000..8bea842 --- /dev/null +++ b/solutions/slim_agents/agents-1-start_here.py @@ -0,0 +1,158 @@ + +""" The objective of this script is to provide several entrypoints into LLMWare models and to start integrating +these models into larger workflows. + + Prereqs: + + `pip3 install llmware` + + This is the 'entrypoint' example that provides a general introduction of llmware models. + + If you have any dependency install issues, please review the README, docs link, or raise an Issue. + + Usually, if there is a missing dependency, the code will give the warning - and a clear + direction like `pip install transformers' required for this example, etc. + + As an alternative to pip install ... if you prefer, you can also clone the repo from github - + which provides a benefit of having access to 100+ examples. + + To clone the repo: + + git clone "https://www.github.com/llmware-ai/llmware.git" + sh "welcome_to_llmware.sh" + + The second script "welcome_to_llmware.sh" will install all of the dependencies. + + If using Windows, then use the "welcome_to_llmware_windows.sh" script. + +""" + + +from llmware.models import ModelCatalog + +# GETTING STARTED - all LLMWare models are accessible through the ModelCatalog generally consisting of two steps +# to access any model + +# Step 1 - load the model - pulls from global repo the first time, and then automatically caches locally +# Step 2 - use the model with inference or function call + +# 'Standard' Models use 'inference' and take a general text input and provide a general text output + +model = ModelCatalog().load_model("bling-answer-tool") +response = model.inference("My son is 21 years old.\nHow old is my son?") + +print("\nresponse: ", response) + +# Optional parameters can improve results +model = ModelCatalog().load_model("bling-phi-3-gguf", temperature=0.0,sample=False, max_output=200) + +# all LLMWare models have been fine-tuned to assume that the input will include a text passage, and that the +# model's main job is to 'read' the passage, and then 'answer' a question based on that information + +text_passage = "The company's stock price increased by $3 after reporting positive earnings." +prompt = "What was the increase in the stock price?" + +response = model.inference(prompt,add_context=text_passage) + +print("\nresponse: ", response) + +# inference models can also be integrated into Prompts - which provide advanced handling for integrating +# with knowledge retrieval, managing source information, and providing fact-checking + +# Discovering other models is easy -> to invoke a model, simply use the 'model_name' and pass in .load_model() +# note: model_names starting with 'bling', 'dragon' and 'slim' are llmware models +# -- we do include other popular models such as phi-3, qwen-2, yi, llama-3, mistral +# -- it is easy to extend the model catalog to include other 3rd party models, including ollama and lm studio +# -- we do support open ai, anthropic, cohere and google api models as well + +all_generative_models = ModelCatalog().list_generative_local_models() +print("\n\nModel Catalog - load model with ModelCatalog().load_model(model_name)") +for i, model in enumerate(all_generative_models): + + model_name = model["model_name"] + model_family = model["model_family"] + + print("model: ", i, model) + +# slim models are 'Function Calling' Models that perform a specialized task and output python dictionaries +# -- by design, slim models are specialists that perform single function +# -- by design, slim models generally do not require any specific 'prompt instructions', but will often accept a +# a "parameter" which is passed to the function. + +model = ModelCatalog().load_model("slim-sentiment-tool") +response = model.function_call("That was the worst earnings call ever - what a disaster.") + +# the 'overall' model response is just a python dictionary +print("\nresponse: ", response) +print("llm_response: ", response['llm_response']) +print("sentiment: ", response['llm_response']['sentiment']) + +# here are several slim models applied against a common earnings extract + +text_passage = ("Here’s what Costco reported for its fiscal second quarter of 2024 compared with what Wall Street " + "was expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings " + "per share: $3.92 vs. $3.62 expected. Revenue: $58.44 billion vs. $59.16 billion expected " + "In the three-month period that ended Feb. 18, Costco’s net income rose to $1.74 billion, or " + "$3.92 per share, compared with $1.47 billion, or $3.30 per share, a year earlier. ") + +# extract model takes a 'key' for a parameter, and looks for the 'value' in the text + +model = ModelCatalog().load_model("slim-extract-tool") + +# the general structure of a function call includes a text passage input, a function and parameters +response = model.function_call(text_passage,function="extract",params=["revenue"]) + +print("\nextract response: ", response) + +# topics model will generate a 1-2 word summary that captures the key topic of the passage + +model = ModelCatalog().load_model("slim-topics-tool") +response = model.function_call(text_passage,function="classify", params=["topic"]) + +print("topics response: ", response) + +# tags model will generate a list of key words that can be used as 'tags' to summarize and lookup the information +model = ModelCatalog().load_model("slim-tags-tool") +response = model.function_call(text_passage,function="classify", params=["tags"]) + +print("tags response: ", response) + +# xsum model generates a 'headline' summary (e.g., 'extreme summarization') +model = ModelCatalog().load_model("slim-xsum-tool") + +# where the parameters can be inferred, they are optional +response = model.function_call(text_passage) + +print("xsum (extreme summarization) response - ", response) + +# boolean model answers a question with Yes/No, and is provided an optional '(explain)' will provide an explanation +model = ModelCatalog().load_model("slim-boolean-tool") +response = model.function_call(text_passage,params=["did earnings beat expectations? (explain)"]) +print("boolean yes-no response - ", response) + +# Function calling models generally come with a test set that is a great way to learn how they work +# --please note that each test can take a few minutes with 20-40 test questions + +ModelCatalog().tool_test_run("slim-topics-tool") +ModelCatalog().tool_test_run("slim-tags-tool") +ModelCatalog().tool_test_run("slim-emotions-tool") +ModelCatalog().tool_test_run("slim-summary-tool") +ModelCatalog().tool_test_run("slim-xsum-tool") + +# Function calling models can be integrated into Agent processes which can orchestrate processes comprising multiple +# models and steps - most of our use cases will use the function calling models in that context + +# Last note: most of the models are packaged as "gguf" - usually identified as GGUFGenerativeModel, or +# with '-gguf' or '-tool' at the end of their name. These models are optimized to run most efficiently on +# a CPU-based laptop (especially Mac OS). You can also try the standard Pytorch versions of these models, +# which should yield virtually identical results, but will be slower. + +# Check out some of the other examples in the other files for more details and building off these recipes + +# NEW - if you are running on Windows machines, then we would encourage you to also check out the following two examples +# -- using_openvino - /examples/Models/using_openvino_models.py +# -- using_onnx - /examples/Models/using_onnx_models.py + +# Welcome to LLMWare! + + diff --git a/solutions/slim_agents/agents-10-using_semantic_reranker_with_rag.py b/solutions/slim_agents/agents-10-using_semantic_reranker_with_rag.py new file mode 100644 index 0000000..42a1db9 --- /dev/null +++ b/solutions/slim_agents/agents-10-using_semantic_reranker_with_rag.py @@ -0,0 +1,78 @@ + +""" This example illustrates the use of a reranker model to provide a fast, effective 'in-memory' semantic +similarity, replacing an explicit retrieval using either text or embedding from a vector db. + + This example uses a common pattern of trying to answer a specific factual question from a set of PDF documents. + + These are the key steps of the example: + + 1. Sample agreements are pulled from repo, cached locally, and then iterated. + + 2. Each PDF document (~10-15 pages) is parsed into text chunks in memory. + -- this generates a list of dictionaries, with each dictionary entry consisting of the "text" and metadata + + 3. The reranker model takes as inference input both the query and the full set of text chunks. + + 4. The reranker model returns as output a sorted list of the (top) original text chunk dictionaries + + 5. The top text chunks are added as source to the prompt + + 6. Prompt_with_Sources run with the original query and the concatenated source to answer the question. + + +""" + +import os + +from llmware.parsers import Parser +from llmware.models import ModelCatalog +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def rag_in_memory_with_reranker(): + + """ Executes a rag process in memory using semantic reranker model and bling-phi-3-gguf to answer the question. """ + + query = "What is the annual rate of the executive's base salary?" + + sample_files_path = Setup().load_sample_files(over_write=False) + contracts_path = os.path.join(sample_files_path, "Agreements") + + files = os.listdir(contracts_path) + + # will use two models for the example - reranker + a 'question-answer' rag llm + reranker_model = ModelCatalog().load_model("jina-reranker-turbo") + prompter = Prompt().load_model("bling-phi-3-gguf", temperature=0.0, sample=False) + + for i, doc in enumerate(files): + + if doc.endswith(".pdf"): + + print("\nPROCESSING: ", i, doc) + + parser_output = Parser().parse_one(contracts_path,doc,save_history=False) + + output = reranker_model.inference(query,parser_output,top_n=10, relevance_threshold=0.25) + + use_top = 3 + if len(output) > use_top: + output = output[0:use_top] + + for i, results in enumerate(output): + print("semantic ranking results: ", i, results["rerank_score"], results["text"]) + + sources = prompter.add_source_query_results(output) + responses = prompter.prompt_with_source(query,prompt_name="default_with_context") + + for i, resp in enumerate(responses): + print("\nllm answers: ", i, resp) + + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + rag_in_memory_with_reranker() diff --git a/solutions/slim_agents/agents-11-gguf_streaming.py b/solutions/slim_agents/agents-11-gguf_streaming.py new file mode 100644 index 0000000..3162358 --- /dev/null +++ b/solutions/slim_agents/agents-11-gguf_streaming.py @@ -0,0 +1,55 @@ + +""" This example illustrates how to use the stream method for GGUF models for fast streaming of inference, +especially for real-time chat interactions. + + Please note that the stream method has been implemented for GGUF models starting in llmware-0.2.13. This will be +any model with GGUFGenerativeModel class, and generally includes models with names that end in "gguf". + + See also the chat UI example in the UI examples folder. + + We would recommend using a chat optimized model, and have included a representative list below. + + +""" + + +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs + +# sets an absolute output maximum for the GGUF engine - normally set by default at 256 +GGUFConfigs().set_config("max_output_tokens", 1000) + +chat_models = ["phi-3-gguf", + "llama-2-7b-chat-gguf", + "llama-3-instruct-bartowski-gguf", + "openhermes-mistral-7b-gguf", + "zephyr-7b-gguf", + "tiny-llama-chat-gguf", + "qwen2-7B-instruct-gguf"] + +model_name = chat_models[-1] + +# maximum output can be set optionally at any number up to the "max_output_tokens" set +model = ModelCatalog().load_model(model_name, max_output=500) + +text_out = "" + +token_count = 0 + +# prompt = "I am interested in gaining an understanding of the banking industry. What topics should I research?" +prompt = "What are the benefits of small specialized LLMs?" + +# since model.stream provides a generator, then use as follows to consume the generator + +for streamed_token in model.stream(prompt): + + text_out += streamed_token + if text_out.strip(): + print(streamed_token, end="") + + token_count += 1 + +# final output text and token count + +print("\n\n***total text out***: ", text_out) +print("\n***total tokens***: ", token_count) diff --git a/solutions/slim_agents/agents-12-web_services_slim_fx.py b/solutions/slim_agents/agents-12-web_services_slim_fx.py new file mode 100644 index 0000000..52abd44 --- /dev/null +++ b/solutions/slim_agents/agents-12-web_services_slim_fx.py @@ -0,0 +1,203 @@ + +""" This example illustrates a more complex recipe to generate a structured financial research dictionary with + 30 keys and values produced, using a combination of models and web services: + + Models + 1. slim-extract-tool + 2. slim-summary-tool + 3. bling-stablelm-3b-tool + + Web Services + 1. Yfinance - stock ticker + 2. Wikipedia - company background information + + The example shows how to extract keys from one source that can then be used as a lookup in a web service to + supplement the original source materials, and provide a secondary source, which can then also be prompted and + used to extract, analyze and summarize key information. + + NOTE: to run this example, please install yfinance library, e.g., 'pip3 install yfinance' + + """ + + +from llmware.web_services import YFinance +from llmware.models import ModelCatalog +from llmware.parsers import WikiParser + +from importlib import util +if not util.find_spec("yfinance"): + print("\nto run this example, you need to install yfinance first, e.g., pip3 install yfinance") + +# our input - financial news article + +text=("BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its " + "third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter " + "of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* " + "at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis " + "NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis " + "Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin " + "increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges " + "Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring " + "charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these " + "charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to " + "drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by " + "the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and " + "work with our wholesale partners to elevate and grow the marketplace.") + + +def research_example1(): + + """ End-to-end example generating 30 output key:value pairs """ + + # load three models in this example + + model = ModelCatalog().load_model("slim-extract-tool", temperature=0.0, sample=False) + model2 = ModelCatalog().load_model("slim-summary-tool", sample=False,temperature=0.0,max_output=200) + model3 = ModelCatalog().load_model("bling-stablelm-3b-tool", sample=False, temperature=0.0) + + research_summary = {} + + # extract information from the source materials + + extract_keys = ["stock ticker", "company name", + "total revenues", "restructuring charges", + "digital growth", "ceo comment", "quarter end date"] + + print("\nStep 1 - extract information from source text\n") + + for keys in extract_keys: + response = model.function_call(text,params=[keys]) + dict_keys = keys.replace(" ", "_") + print(f"update: extracting - {keys} - {response['llm_response']}") + if dict_keys in response["llm_response"]: + value = response["llm_response"][dict_keys][0] + research_summary.update({dict_keys: value}) + else: + print("could not find look up key successfully - ", response["llm_response"]) + + # secondary lookups using extracted information + + print("\nStep 2 - use extracted stock ticker in web service lookup to YFinance\n") + + if "stock_ticker" in research_summary: + ticker = research_summary["stock_ticker"] + # a little kludge related to yfinance api + ticker_core = ticker.split(":")[-1] + + yf = YFinance().get_stock_summary(ticker=ticker_core) + print("yahoo finance stock info: ", yf) + + research_summary.update({"current_stock_price": yf["currentPrice"]}) + research_summary.update({"high_ltm": yf["fiftyTwoWeekHigh"]}) + research_summary.update({"low_ltm": yf["fiftyTwoWeekLow"]}) + research_summary.update({"trailing_pe": yf["trailingPE"]}) + research_summary.update({"forward_pe": yf["forwardPE"]}) + research_summary.update({"volume": yf["volume"]}) + + yf2 = YFinance().get_financial_summary(ticker=ticker_core) + print("yahoo finance fin info - ", yf2) + research_summary.update({"market_cap": yf2["marketCap"]}) + research_summary.update({"price_to_sales": yf2["priceToSalesTrailing12Months"]}) + research_summary.update({"revenue_growth": yf2["revenueGrowth"]}) + research_summary.update({"ebitda": yf2["ebitda"]}) + research_summary.update({"gross_margin": yf2["grossMargins"]}) + research_summary.update({"currency": yf2["currency"]}) + + yf3 = YFinance().get_company_summary(ticker=ticker_core) + print("yahoo finance company info - ", yf3) + research_summary.update({"sector": yf3["sector"]}) + research_summary.update({"website": yf3["website"]}) + research_summary.update({"industry": yf3["industry"]}) + research_summary.update({"employees": yf3["fullTimeEmployees"]}) + + execs = [] + if "companyOfficers" in yf3: + for entries in yf3["companyOfficers"]: + if "totalPay" in entries: + pay = entries["totalPay"] + else: + pay = "pay-NA" + + if "age" in entries: + age = entries["age"] + else: + age = "age-NA" + + execs.append((entries["name"], entries["title"], age, pay)) + research_summary.update({"officers": execs}) + + print("\nStep 3 - use extracted company name to lookup in Wikipedia web service - and add background data\n") + + if "company_name" in research_summary: + + company_name = research_summary["company_name"] + output = WikiParser().add_wiki_topic(company_name, target_results=1) + + # get company summary + company_overview = "" + for i, blocks in enumerate(output["blocks"]): + if i < 3: + company_overview += blocks["text"] + + # call summary model to summarize + print("-- calling summary model to summarize the first part of the Wikipedia article") + summary = model2.function_call(company_overview, params=["company history (5)"]) + print("-- slim-summary - summary (5 points): ", summary) + + research_summary.update({"summary": summary["llm_response"]}) + + # get founding date + print("\n-- calling extract model to get key piece of information from the Wikipedia article - company founding date") + response = model.function_call(company_overview, params=["founding date"]) + print("-- slim-extract - founding date: ", response) + + research_summary.update({"founding_date": response["llm_response"]["founding_date"][0]}) + + print("\n-- calling extract model to get a short company business") + response = model.function_call(company_overview, params=["company description"]) + print("-- slim-extract - response: ", response) + research_summary.update({"company_description": response["llm_response"]["company_description"][0]}) + + # ask other questions directly + print("\n-- asking a question directly to the Wikipedia article about the business") + response = model3.inference("What is an overview of company's business?", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"business_overview": response["llm_response"] }) + + print("\n-- asking a question about the origin of the company's name") + response = model3.inference("What is the origin of the company's name?", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"origin_of_name": response["llm_response"]}) + + print("\n-- asking a question about the company's products") + response = model3.inference("What are the product names", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"products": response["llm_response"]}) + + print("\n\nStep 4 - Completed Research - Summary Output\n") + print("research summary: ", research_summary) + + item_counter = 1 + + for keys, values in research_summary.items(): + if isinstance(values, str): + + values = values.replace("\n", "") + values = values.replace("\r", "") + values = values.replace("\t", "") + + print(f"\t\t -- {item_counter} - \t - {keys.ljust(25)} - {str(values).ljust(40)}") + item_counter += 1 + + return research_summary + + +if __name__ == "__main__": + + research_example1() + + + + + + diff --git a/solutions/slim_agents/agents-13-text2sql-end-to-end-2.py b/solutions/slim_agents/agents-13-text2sql-end-to-end-2.py new file mode 100644 index 0000000..6ce7d2d --- /dev/null +++ b/solutions/slim_agents/agents-13-text2sql-end-to-end-2.py @@ -0,0 +1,113 @@ + +""" This example shows an end-to-end recipe for querying SQL database using only natural language. + + The example shows the following steps: + + 1. Loading "slim-sql-tool" and running initial tests to confirm installation. + 2. Generating a SQL table from a sample CSV file included with the slim-sql-tool install. + 3. Asking basic natural language questions: + A. Looks up the table schema + B. Packages the table schema with query + C. Runs inference to convert text into SQL + D. Queries the database with the generated SQL + E. Returns result + 3. All work performed on an integrated 'llmware-sqlite-experimental.db' that can be deleted safely anytime + as part of experimentation lifecycle. + + UPDATE: please see also the related example in Use_Cases/agent_with_custom_tables.py, which illustrates a more + generalized version of this script running on Postgres. + +""" + +import os + +from llmware.agents import SQLTables, LLMfx +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + + +def sql_e2e_test_script(table_name="customers1",create_new_table=False): + + """ This is the end-to-end execution script. """ + + # create table if needed to set up + if create_new_table: + + # looks to pull sample csv 'customer_table.csv' from slim-sql-tool model package files + sql_tool_repo_path = os.path.join(LLMWareConfig().get_model_repo_path(), "slim-sql-tool") + + if not os.path.exists(sql_tool_repo_path): + ModelCatalog().load_model("llmware/slim-sql-tool") + + files = os.listdir(sql_tool_repo_path) + csv_file = "customer_table.csv" + + if csv_file in files: + + # to create a testing table from a csv + sql_db = SQLTables(experimental=True) + sql_db.create_new_table_from_csv(sql_tool_repo_path, csv_file, table_name=table_name) + # end - creating table + + print("update: successfully created new db table") + else: + print("something has gone wrong - could not find customer_table.csv inside the slim-sql-tool file package") + + # query starts here + agent = LLMfx() + agent.load_tool("sql", sample=False, get_logits=True, temperature=0.0) + + # Pass direct queries to the DB + + query_list = ["Which customers have vip customer status of yes?", + "What is the highest annual spend of any customer?", + "Which customer has account number 1234953", + "Which customer has the lowest annual spend?", + "Is Susan Soinsin a vip customer?"] + + for i, query in enumerate(query_list): + + # query_db method is doing all of the work + # -- looks up the table schema in the db using the table_name + # -- packages the text-2-sql query prompt + # -- executes sql method to convert the prompt into a sql query + # -- attempts to execute the sql query on the db + # -- returns the db results as 'research' output + + response = agent.query_db(query, table=table_name) + + for x in range(0,len(agent.research_list)): + print("research: ", x, agent.research_list[x]) + + return 0 + +def delete_table(table_name): + + """ Start fresh in testing - delete table in experimental local SQLite DB """ + + sql_db = SQLTables(experimental=True) + sql_db.delete_table(table_name, confirm_delete=True) + + return True + + +def delete_db(): + + """ Start fresh in testing - deletes SQLite DB and starts over. """ + + sql_db = SQLTables(experimental=True) + sql_db.delete_experimental_db(confirm_delete=True) + + return True + + +if __name__ == "__main__": + + ModelCatalog().get_llm_toolkit(tool_list=["sql"]) + + # run an end-to-end test + sql_e2e_test_script(table_name="customer1",create_new_table=True) + + # third - delete and start fresh for further testing + delete_table("customer1") + diff --git a/solutions/slim_agents/agents-14-rag_instruct_benchmark_tester.py b/solutions/slim_agents/agents-14-rag_instruct_benchmark_tester.py new file mode 100644 index 0000000..5ae3d01 --- /dev/null +++ b/solutions/slim_agents/agents-14-rag_instruct_benchmark_tester.py @@ -0,0 +1,109 @@ + +""" This example shows how to run the RAG benchmark test against a selected model in the Model Catalog, and +generate a json test report. """ + +import os +import json +import time + +from llmware.prompts import Prompt +from llmware.configs import LLMWareConfig + +from datasets import load_dataset + + +def load_rag_benchmark_tester_dataset(): + + """ Pulls the standard 200 question RAG benchmark test from llmware Huggingface repository. """ + + dataset_name = "llmware/rag_instruct_benchmark_tester" + print(f"\n > Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + + +def model_test_run_general(model_name, prompt_list, save_fp=None, test_name_base=None): + + # use llmware_path if no path provided + + if not save_fp: + save_fp = LLMWareConfig().get_llmware_path() + + if not test_name_base: + test_name_base = model_name + + # run direct inference on model + print("\nupdate: Starting Generative Instruct Custom Fine-tuned Test") + + t1 = time.time() + + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False, max_output=100) + + total_response_output = [] + answer_sheet = [] + + for i, entries in enumerate(prompt_list): + + prompt = entries["query"] + context = entries["context"] + answer = "" + + if "answer" in entries: + answer = entries["answer"] + + output = prompter.prompt_main(prompt,context=context,prompt_name="default_with_context") + + print("\nupdate: model inference output - ", i, output["llm_response"]) + print("update: gold_answer - ", i, answer) + + core_output = {"number": i, + "llm_response": output["llm_response"], + "gold_answer": answer, + "prompt": prompt, + "usage": output["usage"]} + + answer_only = {"number": i, "llm_response": output["llm_response"], + "gold_answer": answer} + + total_response_output.append(core_output) + answer_sheet.append(answer_only) + + t2 = time.time() + + print("update: total processing time: ", t2-t1) + + test_fn = test_name_base + "_core_rag_test.jsonl" + f_out = open(os.path.join(save_fp, test_fn), "w") + + for entry in total_response_output: + jsonl_row = json.dumps(entry) + f_out.write(jsonl_row) + f_out.write("\n") + + f_out.close() + + answer_sheet_fn = test_name_base + "_answer_sheet.jsonl" + f_out = open(os.path.join(save_fp, answer_sheet_fn), "w") + + for entry in answer_sheet: + jsonl_row = json.dumps(entry) + f_out.write(jsonl_row) + f_out.write("\n") + + f_out.close() + + return total_response_output + + +if __name__ == "__main__": + + model_name = "bling-phi-3.5-gguf" + + core_test_set = load_rag_benchmark_tester_dataset() + + output = model_test_run_general(model_name, core_test_set) diff --git a/solutions/slim_agents/agents-15-get_model_benchmarks.py b/solutions/slim_agents/agents-15-get_model_benchmarks.py new file mode 100644 index 0000000..bcb0690 --- /dev/null +++ b/solutions/slim_agents/agents-15-get_model_benchmarks.py @@ -0,0 +1,28 @@ + +""" This example shows how to use the new Model Benchmark lookup to start using benchmark performance test data for +llmware models. """ + +from llmware.model_configs import model_benchmark_data +from llmware.models import ModelCatalog + +# view all benchmark data available +print("\nModel Benchmark Data Available") +for i, model in enumerate(model_benchmark_data): + print("model: ", i, model) + +# lookup data for a specific model +model = "bling-phi-3-gguf" +print(f"\nModel Lookup - {model}") +score = ModelCatalog().get_benchmark_score(model) +print("score: ", score) + +# lookup with a simple filter - models with less than 7B parameters and accuracy_score > 95 +condition = [{"parameters": "parameters < 7"}, {"accuracy_score": "accuracy_score > 95"}] + +accurate_small_models = ModelCatalog().get_benchmark_by_filter(condition) +for a, mod in enumerate(accurate_small_models): + print("accurate models: ", a, mod) + + +# save a copy of the benchmark data to jsonl for future use +ModelCatalog().save_benchmark_report() diff --git a/solutions/slim_agents/agents-2-llmware_model_sampler_bling_dragon.py b/solutions/slim_agents/agents-2-llmware_model_sampler_bling_dragon.py new file mode 100644 index 0000000..874da9f --- /dev/null +++ b/solutions/slim_agents/agents-2-llmware_model_sampler_bling_dragon.py @@ -0,0 +1,524 @@ + +""" WELCOME TO LLMWARE - BLING & DRAGON Model Zoo Sampler + + Get started with local inferences in minutes ... + + This example is a "hello world" model zoo sampler for LLMWare BLING and DRAGON models, and shows a simple + repeatable recipe for prompting models locally using a provided set of sample context passages and questions. + + It is designed to be easy to select among different LLMWARE models in the BLING (Best Little Instruct No-GPU) + and DRAGON (Delivering RAG On ...) model families. + + Also, please feel free to swap out the test passages and questions with your own test set. + + BACKGROUND ON BLING + DRAGON + + All of these models have been fine-tuned on complex business, financial and legal materials, with specific + focus on accurate fact-based question-answering for a context passage. Key training objectives: + + -- Fact-based - rely upon the grounded truth from the context passage, rather than any 'background' implicit + knowledge on the subject. As a result, if no context passage is provided in the prompt, the model will often + respond with "Not Found" or potentially no response. This is less fun for chat applications, but extremely + useful behavior for RAG, Agent and workflow based processes. Hallucinations are rather low as a result, + especially when using `temperature=0.0` and `sample=False` options. + + -- Short Answers - the responses will not be expansive in a pleasing dialog/chat oriented way, but rather + stick to the facts and answer the target question, often times with only a few tokens. This helps with both + programmatic workflows to categorize/sort responses, integrate answers into reports and other automated + aggregations, as well as improving the speed of generation for local inference. + + -- Negative sampling with "Not Found" - the training set includes a well-curated set of negative samples + in which the question can not be answered from the context passage, and rather than "over-interpret", "fill + in the gaps" or provide a lengthy apology message, the model will generally respond with the consistent + "Not Found" message which can be useful in RAG contexts in looking to extract values to identify among + multiple passages which, if any, can answer the target question. + + -- No prompt instructions expected - the model has been fine-tuned to expect a context passage to read, and then + answer a question based on it in a fact-based, grounded way. No need for "You are a helpful assistant" and other + verbiage. + + BLING models vary between 0.5B - 3.8B parameters and have been specifically designed to run on a CPU, including + on local laptops. + + DRAGON models vary between 6-9B parameters, and when quantized, can generally run OK on most CPU-based laptops, + but are designed for optimal use on a GPU or inference server. These models operate on the same principles + as the BLING models, making it easy to 'test' with BLING, and then 'upgrade' to DRAGON in shifting into + production environment for greater accuracy. + + We are always updating the BLING and DRAGON model collection with new models, including improvements in the training + techniques and experimenting with new base models, and try to keep this script updated as a good + 'hello world' sandbox entry point for testing and evaluation. + + We have trained these models on a wide range of base foundation models to also support preferences among specific + users and clients for a particular base model. + + We score each of these models on a RAG benchmark test for accuracy and a number of specialized metrics. Please + see the example "get_model_benchmarks" for a view of this. + +""" + + +import time +from llmware.prompts import Prompt + + +def hello_world_questions(): + + """ Representative test set - we would recommend running this script as a 'hello world' test on the first + try that you use a model, and then adapt the content to your own set of context and questions. + + There is nothing special about these questions, and in fact, you will note that many of the models will get + a couple of answers wrong (especially the ~1B parameter models). The errors are important insights + as you evaluate which models to consider for your use case. + + To adapt this test set, just create your own list with dictionary entries and keys 'query', 'answer' and 'context'. + + """ + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +def llmware_bling_dragon_hello_world (model_name): + + """ Simple inference loop that loads a model and runs through a series of test questions. """ + + t0 = time.time() + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + # please note that by default, we recommend setting temperature=0.0 and sample=False for fact-based RAG + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"],context=entries["context"], prompt_name="default_with_context") + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2-t1} seconds") + + return 0 + + +if __name__ == "__main__": + + bling_pytorch = [ + + # pytorch models - will run fast on GPU, and smaller ones good for CPU only + # note: you will need to install pytorch and transformers to pull and access these models + + "llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3", + "llmware/bling-phi-3.5" + ] + + dragon_pytorch = [ + + # pytorch models - intended for GPU server use - will require pytorch, transformers, and in some cases, + # other dependencies (einops, flash_attn). + + "llmware/dragon-mistral-7b-v0", + "llmware/dragon-yi-6b-v0", + "llmware/dragon-qwen-7b", + "llmware/dragon-llama-7b-v0", + "llmware/dragon-mistral-0.3", + "llmware/dragon-llama-3.1", + "llmware/dragon-deci-7b-v0" + ] + + bling_gguf = [ + + # smaller cpu-oriented models - optimal for running on a CPU + + "bling-phi-3.5-gguf", # **NEW** - phi-3.5 (3.8b) + "bling-answer-tool", # this is quantized bling-tiny-llama (1.1b) + "bling-qwen-0.5b-gguf", # **NEW** - qwen2 (0.5b) + "bling-qwen-1.5b-gguf", # **NEW** - qwen2 (1.5b) + "bling-stablelm-3b-tool", # quantized bling-stablelm-3b (2.7b) + "bling-phi-3-gguf", # quantized phi-3 (3.8b) + "bling-phi-2-gguf", # quantized phi-2 (2.7b) + ] + + dragon_gguf = [ + + # larger models - 6b - 9b + + "dragon-yi-answer-tool", # quantized yi-6b (v1) (6b) + "dragon-llama-answer-tool", + "dragon-mistral-answer-tool", + "dragon-qwen-7b-gguf", # **NEW** qwen2-7b (7b) + "dragon-yi-9b-gguf", # **NEW** yi-9b (8.8b) + "dragon-llama-3.1-gguf", + "dragon-mistral-0.3-gguf" + + ] + + # for most use cases, we would recommend using the GGUF for faster inference + # NEW - if you are running on a Windows machine, then try substituting for one of the following: + # -- "bling-tiny-llama-ov" -> uses OpenVino model version - requires `pip install openvino` and `pip install openvino_genai` + # -- "bling-tiny-llama-onnx" -> uses ONNX model version - requires `pip install onnxruntime_genai` + + my_model = bling_gguf[1] + + llmware_bling_dragon_hello_world(my_model) + + diff --git a/solutions/slim_agents/agents-3-using_slim_extract.py b/solutions/slim_agents/agents-3-using_slim_extract.py new file mode 100644 index 0000000..9287845 --- /dev/null +++ b/solutions/slim_agents/agents-3-using_slim_extract.py @@ -0,0 +1,417 @@ + +""" This example illustrates how to use the slim-extract model to extract custom keys from selected text. + We have included a set of sample earnings releases (comprising lines ~10 - ~385 of this script), and run a + simple loop through the earnings releases, showing how to create an extract prompt to identify + 'revenue growth' from these examples. + + There are several function-calling models in the slim-extract family, fine-tuned on multiple leading + small model base foundations - full list and options are below in the code. """ + +from llmware.models import ModelCatalog + +# Sample earnings releases + +earnings_releases = [ + + {"context": "Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee."}, + + {"context": "Dick’s Sporting Goods raised its dividend by 10% on Thursday as the company posted its largest sales " + "quarter in its history and projected another year of growth. The company’s shares jumped more than " + "15% in intraday trading. CEO Lauren Hobart said on an earnings call Thursday that Dick’s sales " + "growth came from bigger tickets — either higher prices or more expensive items — as its transactions " + "were flat. Many retailers benefited from a 53rd week in fiscal 2023, but Dick’s said it still broke " + "records during its fiscal fourth quarter even without those extra days. Here’s how the athletic " + "apparel retailer did compared with what Wall Street was anticipating, based on a survey of " + "analysts by LSEG, formerly known as Refinitiv: Earnings per share: $3.85 adjusted vs. $3.35 expected " + "Revenue: $3.88 billion vs. $3.80 billion expected The company’s reported net income for the three-month " + "period that ended Feb. 3 was $296 million, or $3.57 per share, compared with $236 million, or $2.60 a " + "share, a year earlier. Excluding one-time items related to impairment charges and inventory write-offs, " + "Dick’s reported earnings per share of $3.85. Sales rose to $3.88 billion, up about 8% from $3.60 billion " + "a year earlier. “With our industry-leading assortment and strong execution, we capped off the year " + "with an incredibly strong fourth quarter and holiday season,” Hobart said in a statement. “We are " + "guiding to another strong year in 2024. We plan to grow both our sales and earnings through " + "positive comps, higher merchandise margin and productivity gains,” she added. During the quarter, " + "same-store sales rose 2.8%, well ahead of the 0.8% lift that analysts had expected, according to " + "StreetAccount. “Growth in transactions” and market share gains drove the increase, said Executive " + "Chairman Ed Stack."}, + + {"context": "Comcast topped both revenue and profit estimates in the fourth quarter as it lost fewer broadband " + "subscribers than expected, and it raised its dividend 7%, the company said Thursday. " + "Here’s how Comcast performed, compared with estimates from analysts surveyed by LSEG, " + "formerly known as Refinitiv. Earnings per share: 84 cents adjusted vs. 79 cents expected " + "Revenue: $31.25 billion vs. $30.51 billion expected For the quarter ended Dec. 31, net " + "income rose 7.8% to $3.26 billion, or 81 cents a share, compared with $3.02 billion, or " + "70 cents a share, a year earlier. Revenue increased 2.3% compared with the prior-year period. " + "Adjusted earnings before interest, taxes, depreciation and amortization (EBITDA) was flat year " + "over year at about $8 billion. 'For the third consecutive year, we generated the highest revenue, " + "adjusted EBITDA and adjusted EPS in our company’s history', Comcast Chief Executive Officer Brian " + "Roberts said in a statement. 'We also reported the highest adjusted EBITDA on record at Theme Parks; " + "were the #1 studio in worldwide box office for the first time since 2015; and maintained Peacock’s " + "position as the fastest growing streamer in the U.S.'"}, + + {"context": "Dollar General forecast annual sales above Wall Street estimates on Thursday, banking on higher " + "demand from inflation-hit customers buying groceries and essentials from the discount retailer’s stores. " + "Shares of the company rose about 6% in early trading, after falling nearly 45% in 2023 on rising costs " + "and stiff competition from bigger retailers. But higher prices and borrowing costs have prompted " + "budget-conscious consumers to cook more meals at home, helping Dollar General record stronger " + "footfall at its outlets as shoppers hunt for lower-margin, needs-based goods, over pricier general " + "merchandise. “With customer traffic growth and market share gains during the quarter, we believe our " + "actions are resonating with customers,” CEO Todd Vasos said in a statement. Vasos’s strategy - to focus " + "on the basics, like more employee presence at stores, greater customer engagement and expanding " + "private-label brands - has helped stabilize Dollar General’s business. Over the last few quarters, " + "Dollar General and rival Dollar Tree have struggled with rising costs linked to their supply " + "chains, labor and raw materials, while facing tough competition from retailers like Walmart " + "and Chinese ecommerce platform Temu. Dollar Tree’s shares fell more than 15% on Wednesday, after it " + "forecast weak sales and profit for 2024 and laid out plans to shutter 970 of its Family Dollar " + "stores. “Dollar General has a much rosier outlook than Dollar Tree... Dollar Tree’s challenges " + "with Family Dollar were years in the making, while Dollar General has embarked on an aggressive " + "effort to add more frozen, refrigerated and fresh produce,” eMarketer senior analyst Zak Stambor said. " + "Dollar General forecast 2024 sales to grow between 6.0% and 6.7%, above analysts’ estimate of 4.4% " + "growth to $40.33 billion, according to LSEG data. It still sees annual per-share profit between " + "$6.80 and $7.55, compared with estimates of $7.55. Its fourth-quarter net sales of $9.86 billion " + "surpassed estimates of $9.78 billion. It also reported an estimate-beating profit of $1.83 per share."}, + + {"context": "Shares of Zara owner Inditex hit record highs on Wednesday according to LSEG data, climbing over 6% during " + "intraday trading after the company announced its 2023 full-year results. As of 11:50 London time, shared " + "were just over 6% higher at 43.58 euros, or $47.69. Sales increased 10.4% to 35.9 billion euros for the " + "year, the company said, signaling this was a record high. Sales grew across all geographic regions and " + "across Inditex’s brands and were “very satisfactory,” both online and in store, the company said. A total of " + "5,692 stores were operational at the end of the year, Inditex said, adding it plans to expand further in " + "2024, including with Zara shops in Los Angeles and Las Vegas. The company also plans to open new distribution " + "centers in 2024 and 2025, as part of a major logistics expansion plan that will cost the company " + "investments of 900 million euros in both years. Net income also reached a fresh high after soaring " + "30.3% from 2022 to reach 5.4 billion euros last year. The company’s gross profit came in at 20.8 billion " + "euros, up 11.9% on the year. “Inditex’s performance in 2023 has been excellent. Our teams have been able to " + "take advantage of the opportunities to keep growing profitably. We are investing to drive future growth and " + "continue to offer an attractive remuneration to shareholders,” Inditex CEO Oscar García Maceiras said in a " + "statement. The Spanish clothing company owns a range of vastly popular brands including household name " + "Zara, as well as Pull & Bear, Bershka, Stradivarius, premium retailer Massimo Dutti and sports and the " + "athleisure-focused Oysho. Zara, including the Zara Home range, was the biggest contributor to sales in " + "2023, followed by Pull & Bear and Massimo Dutti, Inditex said Wednesday. The company also indicated " + "that 2024 was off to a strong start, with sales in constant currency up 11% over the Feb. 1 to March 11 " + "stretch, compared with the same period a year earlier."}, + + {"context": "Oracle reported quarterly earnings on Monday that exceeded Wall Street’s expectations. Shares rose " + "13% in extended trading. Here’s how the company did in the fiscal third quarter ending Feb. 29, compared " + "to estimates by LSEG, formerly known as Refinitiv: Earnings per share: $1.41 adjusted vs. $1.38 expected " + "Revenue: $13.28 billion vs. $13.3 billion expected For the fiscal fourth quarter, Oracle said it expects " + "earnings of $1.62 to $1.66 per share. Analysts were expecting $1.64 in adjusted earnings per share, according " + "to LSEG. Revenue growth will be between 4% and 6% over sales of $13.8 billion a year ago. The midpoint of that " + "range would equal revenue of about $14.5 billion, while analysts were expecting a little more than $14.7 billion. " + "Oracle CEO Safra Catz said the company was committed to hitting previously stated goals of $65 billion in " + "sales by fiscal 2026. “Some of these goals might prove to be too conservative given our momentum,” Catz said. " + "Revenue rose 7% in the quarter from $12.4 billion a year earlier. Net income climbed 27% to $2.4 billion, " + "or 85 cents per share, from $1.9 billion, or 68 cents per share, a year ago. Oracle’s cloud services and " + "license support segment, its largest business, saw sales rise 12% to $9.96 billion, slightly beating " + "StreetAccount consensus expectations of $9.94 billion. The company attributed the rise to strong demand " + "for its artificial intelligence servers. Catz said the company added several “large new cloud " + "infrastructure” contracts during the quarter. The company’s cloud revenue, which is reported as part " + "of the cloud services unit, rose 25% year over year to $5.1 billion, Oracle said. “We signed several large " + "deals this quarter and we have many more in the pipeline,” Catz told investors on the earnings call. " + "Oracle Chairman Larry Ellison cited increased business from Microsoft on the earnings call. " + "“We’re building 20 data centers from Microsoft and Azure. They just ordered three more data centers " + "this week,” Ellison said. The company’s other units didn’t fare as well. Cloud license and on-premise sales " + "declined 3% to $1.26 billion, slightly beating StreetAccount’s forecast. Hardware revenue fell 7% to " + "$754 million, while sales in the company’s services division slid 5% to $1.31 billion, both falling short " + "of StreetAccount expectations. Prior to Monday’s report, Oracle shares were up 8.7% for the year, " + "slightly outperforming the S&P 500."}, + + {"context": "Porsche on Tuesday warned that profitability will decline this year as it launches new models amid " + "tough economic conditions, but hiked its dividend on the back of a rise in 2023 operating profit. The German " + "luxury automaker said it expects an operating return on sales of between 15% and 17% in 2024, down from the " + "18% margin notched in 2023 and 2022. In the long term, the group targets an operating return on sales of more " + "than 20%. Explaining the more cautious profitability outlook, the company cited “the comprehensive renewal " + "of its product range in 2024, the global framework conditions, higher depreciations on capitalized " + "development costs and the continued investments in the brand and the Porsche ecosystem.” The company’s shares " + "were around 4.8% higher by early afternoon, having reversed opening losses of more than 2%. Porsche is " + "launching four new car ranges in 2024 in the form of the Panamera, Macan, Taycan and 911 model lines. " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China WATCH NOW VIDEO 03:17 " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China “2024 is going to be a year of " + "product launches for Porsche – more so than any year in our history,” Chairman Oliver Blume said in a " + "statement. “We will be introducing a variety of exhilarating sports cars to the road, they will delight " + "our customers around the world. This will put the wind at our back for years to come.”"}, + + {"context": "Lego on Tuesday reported its full-year 2023 results, saying it’s revenue grew by 2% throughout the year, " + "in line with expectations. The company made “very, very strong progress” and “grew comfortably” in the " + "U.S., its CEO Niels Christiansen told CNBC. The toy industry has been struggling to maintain " + "pandemic-era growth as inflation is putting pressure on demand and sales. In-store participation is greater " + "than prior to the pandemic, Lego CEO says In-store participation is greater than prior " + "to the pandemic, Lego CEO says The chief executive of Denmark’s Lego on Tuesday reflected on a tough year " + "for the world’s largest toymaker, and outlined the firm’s long-term plans to stay relevant and “cool with kids. ”" + "Lego said its 2023 revenue was 2% higher compared to the previous year, growing to 65.9 billion Danish krone " + "(around $9.65 billion). This was in line with expectations, Lego said in a statement. “It was a difficult year,” " + "Lego CEO Niels Christiansen told CNBC. However, he said the company had “managed to take quite a bit of " + "market share.” The Danish toymaker said operating profit declined slightly from 17.9 billion Danish krone " + "to 17.1 billion, noting that it had boosted spending on strategic initiatives designed to drive growth. " + "Net profit came in at 13.1 billion Danish krone in 2023, compared to 13.8 billion the previous year. " + "Consumer sales were up 4% despite slumping in China, Lego said, attributing the growth to increasing demand " + "in the U.S. and central and eastern Europe. It comes as the wider toy industry has been struggling to " + "maintain growth after booming during the coronavirus pandemic, when parents looked for new ways to " + "entertain their children and adults re-discovered childhood pastimes. Toy company Hasbro earlier this month " + "said its 2023 revenue fell by 15% compared to 2022 and that it expected to see a further decline this year."}, + + {"context": "Adidas on Wednesday warned of a sales decline in its overstocked North American market in 2024, as the " + "German sportswear brand continues to sell off its remaining Yeezy inventory. Currency-neutral sales in " + "North America are expected to decline to a mid-single-digit rate in 2024, but are projected to notch " + "mid-single-digit growth worldwide despite persistent “macroeconomic challenges and geopolitical tensions,” " + "the company said. Adidas confirmed its 2023 operating profit came in at 268 million euros ($292.9 million) " + "on the back of flat currency-neutral sales, significantly above prior expectations as the company continues " + "to take a hit from the cessation of its line of Yeezy — footwear the retailer produced in a collaboration with " + "American rapper Ye, formerly known as Kanye West. For the fourth quarter, the company posted an operating " + "loss of 377 million euros. The board proposed a flat dividend of 0.70 euros per share. “Although by far not " + "good enough, 2023 ended better than what I had expected at the beginning of the year,” CEO Bjørn Gulden " + "said in a statement. “Despite losing a lot of Yeezy revenue and a very conservative sell-in strategy, " + "we managed to have flat revenues. We expected to have a substantial negative operating result, but " + "achieved an operating profit of €268 million.” Adidas was confirming preliminary results released in late " + "January, when it announced that it would not write off the majority of its Yeezy inventory and would instead " + "sell off the remaining shoes at cost. The sportswear giant was forced to axe the Yeezy line after terminating " + "its partnership with Ye over a string of anti-Semitic remarks that the rapper made in 2022. Adidas said the " + "discontinuation of Yeezy represented a drag of around 500 million euros in the year-on-year comparison " + "through 2023, though the sale of parts of the remaining inventory in the second and third quarter positively " + "impacted net sales by around 750 million euros. “With a very disciplined go-to-market and buying process, " + "we reduced our inventories by almost €1.5 billion. With the exception of the U.S., we now have healthy " + "inventories everywhere,” Gulden said. He added that the company is expecting some growth in the " + "first quarter of 2024 and a further pick-up in the second half of the year. “We still have a lot of work " + "to do, but I feel very confident we are on the right track. We will bring adidas back again. Give us some " + "time and we will again say – we got this!” he said. Adidas projected an operating profit of around " + "500 million euros in 2024, with unfavorable currency effects expected to “weigh significantly on the " + "company’s profitability” because of adverse impacts on both reported revenues and gross margin development." + "Adidas shares were flat by mid-morning on Wednesday. Mamta Valechha, equity research analyst at " + "Quilter Cheviot, said that, given that the headline numbers were already pre-released in January, the most " + "interesting aspect of Wednesday’s report was the “clear acceleration of the Adidas brand.”"}, + + {"context": "Costco on Thursday missed Wall Street’s revenue expectations for its holiday quarter, despite reporting " + "year-over-year sales growth and strong e-commerce gains. Shares of the retailer fell about 4% in aftermarket " + "trading. The company’s stock had hit a 52-week high earlier in the day. Here’s what Costco reported for its " + "fiscal second quarter of 2024 compared with what Wall Street was expecting, based on a survey of analysts by " + "LSEG, formerly known as Refinitiv: Earnings per share: $3.92 vs. $3.62 expected Revenue: $58.44 billion vs. " + "$59.16 billion expected In the three-month period that ended Feb. 18, Costco’s net income rose to " + "$1.74 billion, or $3.92 per share, compared with $1.47 billion, or $3.30 per share, a year earlier. " + "Costco’s revenue for the quarter increased from $55.27 billion in the year-ago period. Comparable sales for " + "the company increased 5.6% year over year and 4.3% in the U.S. Excluding changes in gas prices and foreign " + "currency, the metric increased 5.8% overall and 4.8% in the U.S. Sales of food and sundries, a category " + "that includes snack foods and beverages, were up by mid single digits in the quarter, CFO Richard Galanti " + "said on the company’s earnings call. Fresh foods were up high single digits and nonfoods were up mid single " + "digits. Ancillary businesses, which includes more service-related purchases like travel, were up by low " + "single digits, he said. Costco’s food court, pharmacy and optical centers were top performers in the quarter " + "and gas was down low single digits as the price per gallon fell. More shoppers came to Costco, and they " + "spent more on their shopping trips during the quarter. Traffic increased 5.3% across the globe and 4.3% in " + "the U.S., Galanti said on the earnings call. The average ticket increased in the U.S. and worldwide, he " + "said. Inflation was roughly flat year over year in the quarter, which allowed the retailer to reduce " + "prices for some items, Galanti said. For example, he said, it’s been able to cut the price of reading " + "glasses from $18.99 to $16.99 and slash the price of a 48 count of Kirkland Signature batteries from " + "$17.99 to $15.99. In the prior quarter, he said inflation was as much as 1% year over year. Galanti said many " + "new items in categories like sporting goods and lawn and garden will also have lower prices compared with " + "a year ago because of falling freight and commodity costs. Costco has 875 warehouses, including 603 in " + "the U.S. and Puerto Rico. It also has clubs in about a dozen other countries, including Canada, Mexico, " + "Japan and China. In the second quarter, Costco opened four new clubs, including three in the U.S. " + "and one in Shenzhen, China. That marked its sixth club to open in China, Galanti said. Two of the three " + "new U.S. locations were Costco Business Centers, which are specifically geared toward small business " + "owners like restaurant operators. As of Thursday’s close, Costco shares have risen nearly 19% since the " + "start of the year. The stock touched a 52-week high of $787.08 earlier in the day and closed at $785.59, " + "bringing the company’s market value to nearly $350 billion."}, + + {"context": "Shares of Teleperformance plunged 23% on Thursday, after the French call center and office services group " + "missed its full-year revenue target and flagged a “volatile economic environment.” Investors have been " + "spooked by the potential impact of artificial intelligence on its business model, as companies become more " + "able to tap into the technology directly for their own benefit. Teleperformance shares dropped 16% last " + "week, according to LSEG data, after Swedish financial services company Klarna said its Open AI-powered " + "customer service assistant was handling two-thirds of customer service calls. But Teleperformance CEO " + "Daniel Julien on Thursday said that AI would be a positive for its business model — and that it will never " + "fully replace the value of human interaction. “AI is part of the solutions we provide to the clients,” " + "Julien told CNBC’s “Squawk Box Europe.” “AI helps to increase the accuracy of our employees ... which is " + "great, but at the end of the day we are here to reduce the friction between the citizens, or the customer, " + "and the companies they have bought a product and service from.” He stressed, “It’s not just a transactional " + "relationship, it has a lot to do with reassuring, with trust, with empathy. So we perceive AI as enhancing " + "the job that our human employees do, but absolutely not replacing them.” hide content Teleperformance SE " + "RT Quote | Exchange | EUR 87.16 quote price arrow up+0.68 (+0.79%) Last | 03/15/24 CET WATCHLIST + QUOTE DETAILS " + "Teleperformance share price. Teleperformance reported 2.3% higher revenue at 8.345 billion euros " + "($9.091 billion) in 2023, as net profit fell year-on-year from 643 million euros to 602 million euros. " + "Diluted earnings per share hit 10.18 euros, down from 10.77 euros. In its results, the company said it is " + "working with clients on 250 AI projects, including in generative AI, and it has expanded its portfolio " + "with new partnerships in the space. “Even the most high-tech or the most AI-involved companies are clients " + "of Teleperformance. We chose that there is a complementarity and not separation,” Julien told CNBC, " + "flagging the company’s agreement with tech giant and major AI player Microsoft. “They are there to provide " + "a solution that is going to augment the productivity, augment the quality of the information that can be " + "given to the customer, but, at the end of the day, the customer is a human being. The day the customer is " + "going to be a robot, maybe AI will replace the humans.”"}, + + {"context": "CrowdStrike shares surged as much as 21% in after-hours trading Tuesday after the cybersecurity " + "company reported a beat on the top and bottom lines, plus issued stronger-than-expected guidance for " + "the upcoming quarter and full year. Here’s how the company did compared to consensus estimates " + "based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per share: 95 cents " + "adjusted vs. 82 cents expected Revenue: $845 million vs. $839 million expected For the period that " + "ended Jan. 31, CrowdStrike saw net income of $54 million, or 22 cents per share, from a " + "$48 million loss, or a 20 cent loss per share, in the year-ago period. CrowdStrike has now " + "reported GAAP net income for the past four quarters, Chief Financial Officer Burt Podbere " + "said in the earnings release. Full-year revenue rose 36% year over year, from $2.24 billion " + "to $3 billion. The company also announced it would acquire Flow Security for an undisclosed " + "price in a cash-and-stock deal, slated to close in the company’s fiscal first quarter. " + "The company has been stepping up its merger and acquisition activity in recent months. “CrowdStrike " + "is cybersecurity’s consolidator of choice, innovator of choice, and platform of choice to " + "stop breaches,” co-founder and CEO George Kurtz said in a release. The company also guided to " + "fiscal first-quarter revenue between $902 million and $906 million, better than a consensus " + "estimate of $899 million. CrowdStrike also expects earnings per share for the period between " + "89 cents and 90 cents, better than the consensus estimate of 82 cents. Podbere also reiterated " + "the company’s focus on achieving $10 billion in annual recurring revenue by 2030. The company " + "reached $3.4 billion in annual recurring revenue in January."}, + + {"context": "Shares of Amer Sports, the maker of Wilson tennis rackets and Lousiville Slugger baseball " + "bats, fell on Tuesday after the company reported strong sales in China but a slowdown in " + "wholesale orders. Here’s how the newly public athletic company did in its fourth quarter. " + "CNBC didn’t compare the results to Wall Street estimates because it’s the first earnings " + "report since Amer Sports went public. Loss per share: 25 cents Revenue: $1.32 billion In the " + "three months ended Dec. 31, the company reported a net loss of $94.9 million, or 25 cents " + "per share, compared with $148.3 million, or 39 cents per share, a year earlier. Sales rose to " + "$1.32 billion, up about 10% from $1.2 billion a year earlier. Shares closed about 5% lower. " + "Amer, which also owns Arc’teryx, Salomon, and a number of other athletic equipment and " + "apparel brands, operates in three distinct business segments. They are technical apparel, " + "which includes its pricey Arc’teryx winter jackets; outdoor performance, such as Salomon’s " + "winter sports equipment; and ball and racquet sports, which includes equipment and apparel " + "from Wilson and Louisville, among others. During the quarter, sales for Amer’s technical " + "apparel rose 26% year over year to $550 million, driven by a 42% jump in direct sales. " + "Sales in the segment primarily come from shoppers who are buying directly from Amer’s " + "brands rather than from wholesale partners. Sales for outdoor performance increased 2% to " + "$523 million, driven by strength in the segment’s winter sports equipment franchise, " + "which was offset by a slowdown in wholesale orders for Salomon footwear. Ball and racquet sales " + "declined 3% to $242 million as the segment lapped tougher comps. In the year-ago period, " + "retailers were still dealing with supply chain issues and had over-ordered equipment like " + "tennis rackets and baseball bats. As they looked to keep their inventory levels in check, " + "some wholesalers pulled back on orders during the quarter, but Amer expects the segment " + "will level out in the quarters ahead and end fiscal 2024 with sales up in the low- to " + "mid-single digit range. The company started trading on the New York Stock Exchange last " + "month under the ticker “AS.” The shares rose just 3% in Amer’s debut on the public " + "markets after it priced its IPO at a discount. Sellers showed muted interest in the " + "stock during its first day of trading over concerns about its connections and exposure to " + "China and its debt-laden balance sheet. Founded in Helsinki in 1950, Amer was a Finnish public " + "company until it was taken private in 2019 by a consortium of investors led by China’s Anta " + "Sports, FountainVest Partners, Anamered Investments and Tencent. Since the acquisition, " + "sales grew about 45% from $2.45 billion in 2020 to $3.55 billion in 2022. Revenue jumped " + "again in 2023 to $4.37 billion, the company said Tuesday."}, + + {"context": "Shares of Dell Technologies popped more than 15% during extended trading Thursday after the company " + "released fourth-quarter results that beat analysts’ estimates and showed strong demand for its " + "artificial intelligence servers. Here’s how the company did: Earnings per share: $2.20 adjusted vs. " + "$1.73 expected by LSEG, formerly known as Refinitiv Revenue: $22.32 billion vs. $22.16 billion " + "expected by LSEG Dell’s revenue for the fiscal 2024 fourth quarter fell 11% from $25.04 billion " + "in the year-ago quarter. The company reported net income $1.16 billion, up 89% from the $614 million " + "it posted in the same period last year. Chief Financial Officer Yvonne McGill said in a release " + "that the company is increasing its annual dividend by 20% to $1.78 per share, which she called " + "a “testament to our confidence in the business.” Dell’s Infrastructure Solutions Group (ISG) " + "reported $9.3 billion in revenue for the quarter, down 6% year over year but up 10% from the " + "third quarter. Servers and networking revenue made up the bulk of that, with $4.9 billion in " + "revenue driven by “AI-optimized servers.” Storage revenue came in at $4.5 billion. The company’s " + "Client Solutions Group (CSG) reported $11.7 billion for the quarter, down 12% year over year. " + "That includes $9.6 billion in commercial client revenue, which fell 11% since the fourth quarter " + "of last year, and $2.2 billion in consumer revenue, down 19% year over year. “Our strong AI-optimized " + "server momentum continues, with orders increasing nearly 40% sequentially and backlog nearly " + "doubling, exiting our fiscal year at $2.9 billion,” Chief Operating Officer Jeff Clarke " + "said in the release. For its first quarter, Dell said during its quarterly call with " + "investors that it expects to report revenue between $21 billion and $22 billion. The company " + "said it is encouraged by momentum around AI, and that it expects to return to growth for " + "fiscal 2025. However, the company noted that the macroeconomic environment is causing some " + "customers to be cautious about infrastructure costs."}, + + {"context": "Birkenstock on Thursday beat holiday quarter revenue expectations, reporting a 22% year-on-year " + "jump, as the German sandal company benefited from higher pricing and rising U.S. demand. As a newly " + "public company, Birkenstock is still getting into a public reporting rhythm and only just " + "released its fiscal 2023 results and 2024 guidance a little over a month ago. On Thursday, " + "it said it stands by guidance issued then and still expects sales to be between 1.74 billion " + "euros and 1.76 billion euros ($1.89 billion and $1.91 billion), representing growth of 17% to 18%. " + "The shoemaker, which started trading on the New York Stock Exchange under the ticker “BIRK” in " + "October, saw a muted debut when it first hit the public markets, with shares sliding more than " + "12% on its first day as a public company. The stock has since rebounded and is up more than 5% " + "this year, as of the Wednesday close. Birkenstock’s shares closed more than 2% lower Thursday. " + "Here’s how the shoemaker did in its fiscal first quarter compared with what Wall Street was " + "anticipating, based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per " + "share: 9 euro cents adjusted vs. 9 euro cents expected. Revenue: 302.9 million euros vs. 288.7 " + "million euros expected. The company reported a net loss of 7.15 million euros for the " + "three-month period that ended Dec. 31, or a loss of 4 euro cents per share. A year earlier, " + "it reported a loss of 9.19 million euros, or a loss of 5 euro cents per share. " + "Excluding one-time items, Birkenstock reported a profit of 17 million euros, or " + "9 euro cents per share. Sales rose to 302.9 million euros, up 22% from 248.5 million euros " + "a year earlier. Adjusted earnings before interest, taxation, depreciation and amortization " + "(EBITDA) rose 12% year on year to 81 million euros, with an adjusted EBITDA margin of 26.9%, " + "down from 29.1% a year earlier. The retailer has been making strides to grow its direct-to-consumer " + "business, which comes with better profits and more customer insights than relying on wholesale partners. " + "CEO Oliver Reichert has said the company deliberately engineers its distribution strategy so " + "demand is higher than supply but it’s working to double its production capabilities over " + "the next three years to narrow that gap. The chief executive said those investments, " + "along with other efforts the company is undertaking to drive growth, is having a “planned” " + "but “temporary” impact to profitability. The company’s gross profit margin inched down to " + "61% from 61.7% during the same period last year, with Birkenstock citing “unfavorable " + "currency translation and the planned, temporary under-absorption from our ongoing " + "capacity expansion.” The company said it continues to carefully track input costs and " + "is mitigating inflationary pressures with “executed, selective price increases.” In Europe, the " + "company said it had “two consecutive price adjustments” with “no signs of rejection.”"}, + + {"context": "Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter."} + +] + +# *** Execution script starts here *** + +# specialized function-calling extraction models on top of several leading small model bases, +# ranging from 0.5b (qwen2) - 3.8b (phi3) + +slim_extract_models = ["slim-extract-tool", # original - stablelm-3b (2.7b) + "slim-extract-tiny-tool", # tiny-llama 1.1b + "slim-extract-qwen-1.5b-gguf", # **NEW** qwen 1.5b + "slim-extract-phi-3-gguf", # **NEW** phi-3 (3.8b) + "slim-extract-qwen-0.5b-gguf"] # **NEW** qwen 0.5b + +# load the model +model = ModelCatalog().load_model("slim-extract-tool",sample=False,temperature=0.0, max_output=100) + +# iterate through the earnings release samples above +for i, sample in enumerate(earnings_releases): + + # key line: execute function_call on selected model with 'custom_key' = "revenue growth" + response = model.function_call(sample["context"], function="extract", params=["revenue growth"]) + + # display the response on the screen + print("extract response: ", i, response["llm_response"]) + diff --git a/solutions/slim_agents/agents-4-using_slim_summary.py b/solutions/slim_agents/agents-4-using_slim_summary.py new file mode 100644 index 0000000..a8768d8 --- /dev/null +++ b/solutions/slim_agents/agents-4-using_slim_summary.py @@ -0,0 +1,45 @@ + +""" This example illustrates how to use slim-summary models to easily create summaries generated as a list of +summary points for easy integration into multi-step workflows. There are several slim-summary function-calling +models available in different sizes and based on leading underlying base models. """ + +from llmware.models import ModelCatalog + +# three slim-summary function calling models available + +slim_summary_models = ["slim-summary-tool", # original - stablelm-3b + "slim-summary-tiny-tool", # small - tiny-llama (1.1b) + "slim-summary-phi-3-gguf" # phi-3 - phi-3 (3.8b) + ] + +# load the model and set the sampling and output parameters +model = ModelCatalog().load_model("slim-summary-tool", + sample=False, + temperature=0.0, + max_output=200) + + +# get the test data set packaged with the model +test_script = ModelCatalog().get_test_script("slim-summary-tool") + +# iterate through the samples +for i, sample in enumerate(test_script): + + # invoke function call on the model, passing the context passage and the 'summarize' function + # the parameter can be a generic phrase, e.g., 'key points' or 'brief description' or 'summary' + # if the material has a lot of numeric data points, try the parameter 'data points' or 'financial data points' + # if you are looking for a single line of output, try 'brief description' + # the number in ( ) is optional - but is intended to guide the model to provide with a list with the requested + # number of elements + + response = model.function_call(sample["context"], function="summarize", params=["data points (5)"]) + + # display the response + print("\nresponse: ", response) + + # check how effectively the model mapped the output to the requested number of points + r = response["llm_response"] + for j, entries in enumerate(r): + print("summary points: ", j, entries) + + diff --git a/solutions/slim_agents/agents-5-agent-llmfx-getting-started.py b/solutions/slim_agents/agents-5-agent-llmfx-getting-started.py new file mode 100644 index 0000000..c526448 --- /dev/null +++ b/solutions/slim_agents/agents-5-agent-llmfx-getting-started.py @@ -0,0 +1,82 @@ + +""" Using SLIM tools as part of an agent workflow - introducing LLMfx class - this example shows how to: + + 1. Create an agent using the LLMfx class. + 2. Load multiple specialized tools for the agent. + 3. Execute a series of function-calls. + 4. Generate a consolidated automatic dictionary report. + +""" + + +from llmware.models import ModelCatalog +from llmware.agents import LLMfx + + +def create_multistep_report(customer_transcript): + + """ Creating a multi-step, multi-model agent workflow """ + + # create an agent using LLMfx class + agent = LLMfx() + + agent.load_work(customer_transcript) + + # load tools individually + agent.load_tool("sentiment") + agent.load_tool("ner") + + # load multiple tools + agent.load_tool_list(["emotions", "topics", "intent", "tags", "ratings", "answer"]) + + # start deploying tools and running various analytics + + # first conduct three 'soft skills' initial assessment using 3 different models + agent.sentiment() + agent.emotions() + agent.intent() + + # alternative way to execute a tool, passing the tool name as a string + agent.exec_function_call("ratings") + + # call multiple tools concurrently + agent.exec_multitool_function_call(["ner","topics","tags"]) + + # the 'answer' tool is a quantized question-answering model - ask an 'inline' question + # the optional 'key' assigns the output to a dictionary key for easy consolidation + agent.answer("What is a short summary?",key="summary") + + # prompting tool to ask a quick question as part of the analytics + response = agent.answer("What is the customer's account number and user name?", key="customer_info") + + # you can 'unload_tool' to release it from memory + agent.unload_tool("ner") + agent.unload_tool("topics") + + # at end of processing, show the report that was automatically aggregated by key + report = agent.show_report() + + # displays a summary of the activity in the process + activity_summary = agent.activity_summary() + + # list of the responses gathered + for i, entries in enumerate(agent.response_list): + print("update: response analysis: ", i, entries) + + output = {"report": report, "activity_summary": activity_summary, "journal": agent.journal} + + return output + + +if __name__ == "__main__": + + # sample customer transcript + + customer_transcript = "My name is Michael Jones, and I am a long-time customer. " \ + "The Mixco product is not working currently, and it is having a negative impact " \ + "on my business, as we can not deliver our products while it is down. " \ + "This is the fourth time that I have called. My account number is 93203, and " \ + "my user name is mjones. Our company is based in Tampa, Florida." + + output = create_multistep_report(customer_transcript) + diff --git a/solutions/slim_agents/agents-6-agent-multistep-analysis.py b/solutions/slim_agents/agents-6-agent-multistep-analysis.py new file mode 100644 index 0000000..9e34dac --- /dev/null +++ b/solutions/slim_agents/agents-6-agent-multistep-analysis.py @@ -0,0 +1,102 @@ + +""" This example shows a complex multi-part research analysis. In this example, we will: + + 1. Build a "research" library. + 2. Query the research library to identify topics of interest. + 3. Create an agent with several analytical tools: sentiment, emotions, topic, entities analysis + 4. Pass the results of our query to the agent to conduct multifaceted analysis. + 5. Apply a top-level filter ('sentiment') on the results from the query + 6. For any of the passages with negative sentiment, we will run a follow-up set of analyses. + 7. Finally, we will assemble the follow-up analysis into a list of detailed reports. +""" + +import os +import shutil + +from llmware.agents import LLMfx +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig +from llmware.setup import Setup + + +def multistep_analysis(): + + """ In this example, our objective is to research Microsoft history and rivalry in the 1980s with IBM. """ + + # step 1 - assemble source documents and create library + + print("update: Starting example - agent-multistep-analysis") + + # note: lines 38-49 attempt to automatically pull sample document into local path + # depending upon permissions in your environment, you may need to set up directly + # if you pull down the samples files with Setup().load_sample_files(), in the Books folder, + # you will find the source: "Bill-Gates-Biography.pdf" + # if you have pulled sample documents in the past, then to update to latest: set over_write=True + + print("update: Loading sample files") + + sample_files_path = Setup().load_sample_files(over_write=False) + bill_gates_bio = "Bill-Gates-Biography.pdf" + path_to_bill_gates_bio = os.path.join(sample_files_path, "Books", bill_gates_bio) + + microsoft_folder = os.path.join(LLMWareConfig().get_tmp_path(), "example_microsoft") + + print("update: attempting to create source input folder at path: ", microsoft_folder) + + if not os.path.exists(microsoft_folder): + os.mkdir(microsoft_folder) + os.chmod(microsoft_folder, 0o777) + shutil.copy(path_to_bill_gates_bio,os.path.join(microsoft_folder, bill_gates_bio)) + + # create library + print("update: creating library and parsing source document") + + LLMWareConfig().set_active_db("sqlite") + my_lib = Library().create_new_library("microsoft_history_0823_1") + my_lib.add_files(microsoft_folder) + + # run our first query - "ibm" + query = "ibm" + search_results = Query(my_lib).text_query(query) + print(f"update: executing query to filter to key passages - {query} - results found - {len(search_results)}") + + # create an agent and load several tools that we will be using + agent = LLMfx() + agent.load_tool_list(["sentiment", "emotions", "topic", "tags", "ner", "answer"]) + + # load the search results into the agent's work queue + agent.load_work(search_results) + + while True: + + agent.sentiment() + + if not agent.increment_work_iteration(): + break + + # analyze sections where the sentiment on ibm was negative + follow_up_list = agent.follow_up_list(key="sentiment", value="negative") + + for job_index in follow_up_list: + + # follow-up 'deep dive' on selected text that references ibm negatively + agent.set_work_iteration(job_index) + agent.exec_multitool_function_call(["tags", "emotions", "topics", "ner"]) + agent.answer("What is a brief summary?", key="summary") + + my_report = agent.show_report(follow_up_list) + + activity_summary = agent.activity_summary() + + for entries in my_report: + print("my report entries: ", entries) + + return my_report + + +if __name__ == "__main__": + + multistep_analysis() + + diff --git a/solutions/slim_agents/agents-7-using-whisper-cpp-sample-files.py b/solutions/slim_agents/agents-7-using-whisper-cpp-sample-files.py new file mode 100644 index 0000000..09939bf --- /dev/null +++ b/solutions/slim_agents/agents-7-using-whisper-cpp-sample-files.py @@ -0,0 +1,81 @@ + +""" This example shows how to use llmware provided sample files for testing with WhisperCPP, integrated as of + llmware 0.2.11. + + # examples - "famous_quotes" | "greatest_speeches" | "youtube_demos" | "earnings_calls" + + -- famous_quotes - approximately 20 small .wav files with clips from old movies and speeches + -- greatest_speeches - approximately 60 famous historical speeches in english + -- youtube_videos - wav files of ~3 llmware youtube videos + -- earnings_calls - wav files of ~4 public company earnings calls (gathered from public investor relations) + + These sample files are hosted in a non-restricted AWS S3 bucket, and downloaded via the Setup method + `load_sample_voice_files`. There are two options: + + -- small_only = True: only pulls the 'famous_quotes' samples + -- small_only = False: pulls all of the samples (requires ~1.9 GB in total) + + Please note that all of these samples have been pulled from open public domain sources, including the + Internet Archives, e.g., https://archive.org. These sample files are being provided solely for the purpose of + testing the code scripts below. Please do not use them for any other purpose. + + To run these examples, please make sure to `pip install librosa` + """ + +import os +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs +from llmware.setup import Setup + +# optional / to adjust various parameters of the model +GGUFConfigs().set_config("whisper_cpp_verbose", "OFF") +GGUFConfigs().set_config("whisper_cpp_realtime_display", True) + +# note: english is default output - change to 'es' | 'fr' | 'de' | 'it' ... +GGUFConfigs().set_config("whisper_language", "en") +GGUFConfigs().set_config("whisper_remove_segment_markers", True) + + +def sample_files(example="famous_quotes", small_only=False): + + """ Execute a basic inference on Voice-to-Text model passing a file_path string """ + + voice_samples = Setup().load_voice_sample_files(small_only=small_only) + + examples = ["famous_quotes", "greatest_speeches", "youtube_demos", "earnings_calls"] + + if example not in examples: + print("choose one of the following - ", examples) + return 0 + + fp = os.path.join(voice_samples,example) + + files = os.listdir(fp) + + # these are the two key lines + whisper_base_english = "whisper-cpp-base-english" + + model = ModelCatalog().load_model(whisper_base_english) + + for f in files: + + if f.endswith(".wav"): + + prompt = os.path.join(fp,f) + + print(f"\n\nPROCESSING: prompt = {prompt}") + + response = model.inference(prompt) + + print("\nllm response: ", response["llm_response"]) + print("usage: ", response["usage"]) + + return 0 + + +if __name__ == "__main__": + + # pick among the four examples: famous_quotes | greatest_speeches | youtube_demos | earnings_calls + + sample_files(example="famous_quotes", small_only=False) + diff --git a/solutions/slim_agents/agents-8-using-phi-3-function-calls.py b/solutions/slim_agents/agents-8-using-phi-3-function-calls.py new file mode 100644 index 0000000..f7f100c --- /dev/null +++ b/solutions/slim_agents/agents-8-using-phi-3-function-calls.py @@ -0,0 +1,99 @@ + +""" This example shows how to use 7 different SLIM function calling models fine-tuned on top of Phi-3: + + -- Extraction - slim-extract-phi-3-gguf - generates python dictionary with 'key' and 'value' + -- Summarization - slim-summary-phi-3-gguf - generates python list with key bullet-point summary + -- XSUM (titles) - slim-xsum-phi-3-gguf - generates python dictionary with 'xsum' key + -- Boolean - slim-boolean-phi-3-gguf - generate python dictionary with "answer" & "explanation" keys + -- Sentiment-NER - slim-sa-ner-phi-3-gguf - generates python dictionary with "sentiment" and selected ner keys + -- Question-Gen - slim-q-gen-phi-3-tool - generates python dictionary with "question" key + -- Question-Answer - slim-qa-gen-phi-3-tool - generates python dictionary with "question" and "answer" key + + The design of these models is to simplify both the input prompt and output to enable easy integration into + programmatic workflows. + + """ + +from llmware.models import ModelCatalog + +# sample text passage that will be used as the basis for the function call analysis + +context_passage = ("Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter.") + + +# for convenience to execute a 'loop', we will set up a dictionary with each function call, and the associated +# model and parameters that are being passed to the model + +phi3_function_call_models = { + + # extract model will look for the 'key' in the params, and return the 'value' found in the text + "extract": {"model": "slim-extract-phi-3-gguf", "params": ["net income"]}, + + # summary model will return a python list with key summary points related to the parameter + "summary": {"model": "slim-summary-phi-3-gguf", "params": ["financial highlights"]}, + + # xsum model produces an 'extreme summarization', e.g. a headline or title + "xsum": {"model": "slim-xsum-phi-3-gguf", "params": ["xsum"]}, + + # boolean model is designed to answer yes/no questions + "boolean": {"model": "slim-boolean-phi-3-gguf", "params": ["Is Best Buy closing stores? (explain)"]}, + + # sentiment-ner model returns several keys for sentiment and ner attributes (e.g., people, place, organization) + "sentiment-ner": {"model": "slim-sa-ner-phi-3-gguf", "params": ["sentiment", "people"]}, + + # q-gen model generates a question from the context passage + "q-gen": {"model": "slim-q-gen-phi-3-tool", "params": ["question"]}, + + # qa-gen model generates question and answer from the context passage + "qa-gen": {"model": "slim-qa-gen-phi-3-tool", "params": ["question, answer"]} + } + +for function, model in phi3_function_call_models.items(): + + print(f"\nfunction: {function} - model - {model['model']} - params - {model['params']}") + slim_model = ModelCatalog().load_model(model["model"], temperature=0.0, sample=False) + + # note: this is the line doing all of the work - each model has been fine-tuned as a 'specialist' for + # its function, so the only required inputs are the source context passage, and the specific parameters to be used + + response = slim_model.function_call(context_passage, params=model["params"]) + + print("response: ", response) + + + diff --git a/solutions/slim_agents/agents-9-document_summarizer.py b/solutions/slim_agents/agents-9-document_summarizer.py new file mode 100644 index 0000000..d71d101 --- /dev/null +++ b/solutions/slim_agents/agents-9-document_summarizer.py @@ -0,0 +1,77 @@ + +""" This Example shows a packaged 'document_summarizer' prompt using the slim-summary-tool. It shows a variety of +techniques to summarize documents generally larger than a LLM context window, and how to assemble multiple source +batches from the document, as well as using a 'query' and 'topic' to focus on specific segments of the document. """ + +import os + +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def test_summarize_document(example="jd salinger"): + + # pull a sample document (or substitute a file_path and file_name of your own) + sample_files_path = Setup().load_sample_files(over_write=False) + + topic = None + query = None + fp = None + fn = None + + if example not in ["jd salinger", "employment terms", "just the comp", "un resolutions"]: + print ("not found example") + return [] + + if example == "jd salinger": + fp = os.path.join(sample_files_path, "SmallLibrary") + fn = "Jd-Salinger-Biography.docx" + topic = "jd salinger" + query = None + + if example == "employment terms": + fp = os.path.join(sample_files_path, "Agreements") + fn = "Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + topic = "executive compensation terms" + query = None + + if example == "just the comp": + fp = os.path.join(sample_files_path, "Agreements") + fn = "Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + topic = "executive compensation terms" + query = "base salary" + + if example == "un resolutions": + fp = os.path.join(sample_files_path, "SmallLibrary") + fn = "N2126108.pdf" + # fn = "N2137825.pdf" + topic = "key points" + query = None + + # optional parameters: 'query' - will select among blocks with the query term + # 'topic' - will pass a topic/issue as the parameter to the model to 'focus' the summary + # 'max_batch_cap' - caps the number of batches sent to the model + # 'text_only' - returns just the summary text aggregated + + kp = Prompt().summarize_document_fc(fp, fn, topic=topic, query=query, text_only=True, max_batch_cap=15) + + print(f"\nDocument summary completed - {len(kp)} Points") + for i, points in enumerate(kp): + print(i, points) + + return 0 + + +if __name__ == "__main__": + + print(f"\nExample: Summarize Documents\n") + + # 4 examples - ["jd salinger", "employment terms", "just the comp", "un resolutions"] + # -- "jd salinger" - summarizes key points about jd salinger from short biography document + # -- "employment terms" - summarizes the executive compensation terms across 15 page document + # -- "just the comp" - queries to find subset of document and then summarizes the key terms + # -- "un resolutions" - summarizes the un resolutions document + + summary_direct = test_summarize_document(example="employment terms") + + diff --git a/solutions/slim_agents/custom_extract_and_lookup.py b/solutions/slim_agents/custom_extract_and_lookup.py new file mode 100644 index 0000000..0a95491 --- /dev/null +++ b/solutions/slim_agents/custom_extract_and_lookup.py @@ -0,0 +1,176 @@ + +""" This example illustrates one of the core function-calling recipes, specifically how to extract a set of values + from a text, using a custom key, and then programmatically use those values as the next step in a lookup + process. There are two different versions included in this script: + + 1. Company name extract and lookup - simple core recipe that can be copied-pasted and adapted. + + 2. Generic extract and lookup - builds on the first recipe and includes some error handling and + parameterizing of key values. Also intended to be a 'copy-paste-adapt' recipe, with highlight on some + of the areas of attention in managing function call outputs programmatically. """ + + +from llmware.models import ModelCatalog +from llmware.parsers import WikiParser + +# our input - financial news article + +text =("BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its " + "third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter " + "of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* " + "at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis " + "NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis " + "Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin " + "increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges " + "Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring " + "charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these " + "charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to " + "drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by " + "the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and " + "work with our wholesale partners to elevate and grow the marketplace.") + + +def company_name_extract_and_lookup(): + + """ This example shows how to perform the following core extract-and-lookup steps: + + 1. Use slim-extract-tool to extract a company name from a piece of text + 2. Use the extracted value programmatically as the basis for a lookup using a web service + """ + + print("\nFirst Example - extract company name and then use as a lookup.\n") + + company_name = "" + output = "" + + # step 1 - run an extract function call on the text + model = ModelCatalog().load_model("slim-extract-tool", temperature=0.0, sample=False) + response = model.function_call(text, function="extract", params=["company name"]) + llm_response = response["llm_response"] + + print("update: llm response: ", llm_response) + + # unpack the output + if "company_name" in llm_response: + company_name = llm_response["company_name"] + if len(company_name) > 0: + company_name = company_name[0] + else: + print(f"no company name found in text - {company_name}") + + print("update: extracted company name: ", company_name) + + # step 2 - use extracted value for lookup + if company_name: + output = WikiParser().add_wiki_topic(company_name, target_results=1) + + if output: + if "articles" in output: + for i, articles in enumerate(output["articles"]): + summary_pp = articles["summary"][0:min(150,len(articles["summary"]))] + + print(f"update: wikipedia articles on {company_name} found: ", i, articles["title"], summary_pp) + + return company_name, output + + +def extract_then_lookup_generalized_example(lookup_key, secondary_key=None): + + """ This example shows the building blocks of using slim-extract function call to perform an extract on + a text, and then programmatically use the extract lookup to perform a research step - with basic unpacking + of the function call model response and error handling. There is an optional secondary_key, which will be used + as an extraction key when and if the secondary material in the lookup step is found. """ + + # objective: find one or more values in a text corresponding to a specific custom key + print(f"\nSecond Example - extract {lookup_key} and then use as a lookup.\n") + + # this is the value that we are looking for + target_value = "" + + # this is the name of the llm response dictionary key derived from the lookup_key + dict_key = lookup_key.replace(" ", "_") + + # step 1 - load the slim-extract-tool model + model = ModelCatalog().load_model("slim-extract-tool", temperature=0.0, sample=False) + + # step 2 - execute a function call with the model, passing our text source and the lookup key + response = model.function_call(text, function="extract", params=[lookup_key]) + llm_response = response["llm_response"] + + if not isinstance(llm_response, dict): + print(f"something has gone wrong, and the model could not produce a function call output.\n" + f"model output: {llm_response})") + return target_value + + if dict_key in llm_response: + + # expect that llm_response will generate a dictionary with key mapping to the params + # e.g., dict_key = "company_name" + + target_value = llm_response[dict_key] + + if isinstance(target_value, list): + + # general form is that the value will be contained in a list, often times in a list with + # a single element consisting of the target value string + + if len(target_value) > 0: + # take the first value in the list + # -- depending upon the query, you may want to keep all of these values for separate lookups + target_value = target_value[0] + else: + # key exception case - the model did not find the lookup key and is returning an empty list + print(f"Lookup key value not found in the text: {lookup_key} - {target_value}") + + else: + # target key structure not found, we can triage in a variety of ways + print(f"update: did not find the target key expected, but was able to extract the following: {llm_response}") + + # for simplicity, we will take the first output value, regardless of the key name + + for keys, values in llm_response.items(): + print(f"llm response: {keys} - {values}") + target_value = values + + if target_value: + if len(target_value) > 0: + target_value = target_value[0] + + if not target_value: + print("update: unfortunately, could not succeed in finding a target value for lookup") + return target_value + + # second step - use the target value for lookup + # -- feel free to replace with a library query or another form of information retrieval + + print(f"update: looked up key - {lookup_key} - and found target value - {target_value} - now using as lookup") + + research_output = WikiParser().add_wiki_topic(target_value, target_results=1) + supplemental_text = research_output["articles"][0]["summary"] + if len(supplemental_text) > 250: + supplemental_text_pp = supplemental_text[0:250] + " ... " + else: + supplemental_text_pp = supplemental_text + + print(f"update: completed research using {lookup_key} - {target_value} - {supplemental_text_pp}") + + output_dict = {"lookup": lookup_key, "target_value": target_value, "research_output": research_output} + + if secondary_key and supplemental_text: + # add a secondary extraction + follow_up_response = model.function_call(supplemental_text, params=[secondary_key]) + print(f"update: follow-up - {target_value} - {secondary_key} - ", follow_up_response["llm_response"]) + + return output_dict + + +if __name__ == "__main__": + + # first example + company_name_extract_and_lookup() + + # second example - a couple of fun prompts to try: + # e.g., #1 - "city" and "population" + # e.g., #2 - "ceo" & "birth date" + output_dict= extract_then_lookup_generalized_example("ceo", secondary_key="birth date") + diff --git a/solutions/slim_agents/document-clustering.py b/solutions/slim_agents/document-clustering.py new file mode 100644 index 0000000..fb3b746 --- /dev/null +++ b/solutions/slim_agents/document-clustering.py @@ -0,0 +1,57 @@ + +"""This example demonstrates the use of LLM function calls to perform document clustering and + automated classification of different parts of a document. """ + +from llmware.parsers import Parser +from llmware.agents import LLMfx +from llmware.setup import Setup + +import os + + +def document_clustering_example (): + + samples_fp = Setup().load_sample_files(over_write=True) + agreements_fp = os.path.join(samples_fp, "Agreements") + agreement_files = os.listdir(agreements_fp) + + if len(agreement_files) == 0: + print("something went wrong") + return -1 + + # parsing the first file (could be random) found in the os.listdir in the Agreements sample folder + contract_chunks = Parser().parse_one_pdf(agreements_fp,agreement_files[0]) + + # create a LLMfx object + agent = LLMfx() + + # there are ~65-70 contract_chunks in ~15 page contract - feel free to slice (faster demo), or the whole thing + agent.load_work(contract_chunks[0:5]) + + agent.load_tool_list(["topics","tags", "ner"]) + + while True: + agent.exec_multitool_function_call(["topics", "tags","ner"]) + + if not agent.increment_work_iteration(): + break + + agent.show_report() + + agent.activity_summary() + + # uncomment this to see a full view of all of the responses + """ + for i, entries in enumerate(agent.response_list): + print("response_list: ", i, entries) + """ + + return agent.response_list + + +if __name__ == "__main__": + + analysis= document_clustering_example() + + + diff --git a/solutions/slim_agents/document_unique_topics_extraction.py b/solutions/slim_agents/document_unique_topics_extraction.py new file mode 100644 index 0000000..0df0483 --- /dev/null +++ b/solutions/slim_agents/document_unique_topics_extraction.py @@ -0,0 +1,61 @@ + +""" +This example illustrates parsing a document and extracting unique topics using the SLIM topics tool +""" + +from llmware.parsers import Parser +from llmware.agents import LLMfx +from llmware.setup import Setup + + + +def document_parser(): + # Add the path to the directory in fp, add the filename in fn + fp = "#Add/Path/To/Document" + fn = "Filename for analysis.pdf" + + + #Given the filename and filepath, parses pdf into chunks + doc_chunks = Parser().parse_one_pdf(fp,fn) + print ("number of chunks: ", len(doc_chunks)) + # create a LLMfx object + agent = LLMfx() + #load in the chunked document. to make the demo run faster or to test, slice it with [0:5] + agent.load_work(doc_chunks) + #load in the topic tool + agent.load_tool_list(["topics"]) + + funcall_list = [] + while True: + funcall_list.append(agent.exec_multitool_function_call(["topics"])) + + if not agent.increment_work_iteration(): + break + + return funcall_list +#Function to collapse the report to show unique topics + + +def collapser(report): + no_duplicates = [] + #raise this to make your program more selective, lower it to get more topics + required_confidence_score = 0.1 + for entry in report: + if entry[0]['confidence_score'] > required_confidence_score: + if 'topics' not in entry[0]['llm_response']: + continue + for topics in entry[0]['llm_response']['topics']: + if topics not in no_duplicates: + no_duplicates.append(topics) + return no_duplicates + + +if __name__ == "__main__": + analysis = document_parser() + print("\n Analysis: Shows Topics located in each chunk of the Document \n") + print(analysis) + print("\n Collapsed Analysis: Shows Unique topics located over the entire Document that meet the required confidence score. \n") + print(collapser(analysis)) + + + diff --git a/solutions/slim_agents/ner-retrieval.py b/solutions/slim_agents/ner-retrieval.py new file mode 100644 index 0000000..3f86341 --- /dev/null +++ b/solutions/slim_agents/ner-retrieval.py @@ -0,0 +1,47 @@ + +""" This example illustrates a common two-step retrieval pattern using a SLIM NER model: + + Step 1: Extract named entity information from a text. In this case, the name of a musician. + Step 2: Use the extracted name information as the basis for a retrieval. In this case, we will use the + extracted named entities to do a lookup in Wikipedia. """ + +from llmware.agents import LLMfx +from llmware.parsers import WikiParser + + +def ner_lookup_retrieval(): + + text = ("The new Miko Marks album is one of the best I have ever heard in a number of years. " + "She is definitely an artist worth exploring further.") + + # create agent + agent = LLMfx() + agent.load_work(text) + agent.load_tool("ner") + named_entities = agent.ner() + ner_dict= named_entities["llm_response"] + + # take named entities found and package into a lookup list + + lookup = [] + for keys, value in ner_dict.items(): + if value: + lookup.append(value) + + for entries in lookup: + + # run a wiki topic query with each of the named entities found + + wiki_info = WikiParser().add_wiki_topic(entries, target_results=1) + + print("update: wiki_info - ", wiki_info) + summary = wiki_info["articles"][0]["summary"] + + print("update: summary - ", summary) + + return 0 + + +if __name__ == "__main__": + + ner_lookup_retrieval() diff --git a/solutions/slim_agents/not_found_extract_with_lookup.py b/solutions/slim_agents/not_found_extract_with_lookup.py new file mode 100644 index 0000000..17d6db6 --- /dev/null +++ b/solutions/slim_agents/not_found_extract_with_lookup.py @@ -0,0 +1,122 @@ + +""" This example illustrates one of the most challenging RAG situations, specifically when a core text does + not provide the answer to a question - will the combination of model and workflow code identify the + 'not found' correctly, and then provide a way to triage. + + In this example, we will use the slim-extract-tool with the following steps: + + 1. Try to obtain the "company founding date" from the first text passage. + 2. Correctly identify that the answer is not in the text. + 3. Programmatically identify and handle the empty [] response. + 4. Based on the empty response condition, trigger a triage process: + + A. Extract a lookup key from the original text, e.g., "company name" + B. Use the lookup key to retrieve a supplemental material from Wikipedia (in this case) + C. Try to extract the "company founding date" from the new materials. + + 5. Provide the correct answer to "company founding date" based on these steps. + + """ + +from llmware.models import ModelCatalog +from llmware.parsers import WikiParser + + +# our input - financial news article - note: the 'company founding date' is not mentioned in the text +# -- our worry is that the model will either 'make up' an answer from background knowledge (which may or may not +# be accurate), or try to incorrectly use a date in the text, e.g, February 29, 2024. + + +text =("BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its " + "third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter " + "of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* " + "at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis " + "NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis " + "Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin " + "increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges " + "Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring " + "charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these " + "charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to " + "drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by " + "the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and " + "work with our wholesale partners to elevate and grow the marketplace.") + + +def not_found_then_triage_lookup(): + + print("\nNot Found Example - if info not found, then lookup in another source.\n") + + extract_key = "company founding date" + dict_key = extract_key.replace(" ", "_") + + company_founding_date = "" + + # step 1 - run an extract function call on the text + model = ModelCatalog().load_model("slim-extract-tool", temperature=0.0, sample=False) + response = model.function_call(text, function="extract", params=[extract_key]) + llm_response = response["llm_response"] + + print(f"update: first text reviewed for {extract_key} - llm response: ", llm_response) + + # unpack the output + if dict_key in llm_response: + + company_founding_date = llm_response[dict_key] + + if len(company_founding_date) > 0: + + # in this case, the value is a list with at least one element, so an 'answer' was found + company_founding_date = company_founding_date[0] + print(f"update: found the {extract_key} value - ", company_founding_date) + return company_founding_date + + else: + + # step 2 - could not find the answer in the original source materials + # e.g., the len of the list associated with the key is zero, or [] + + print(f"update: did not find the target value in the text - {company_founding_date}") + print("update: initiating a secondary process to try to find the information") + + # look up the company name from the original text + response = model.function_call(text, function="extract", params=["company name"]) + + if "company_name" in response["llm_response"]: + company_name = response["llm_response"]["company_name"][0] + + if company_name: + print(f"\nupdate: found the company name - {company_name} - now using to lookup in secondary source") + + # use the company name to lookup materials in Wikipedia secondary source + output = WikiParser().add_wiki_topic(company_name,target_results=1) + + if output: + + supplemental_text = output["articles"][0]["summary"] + + if len(supplemental_text) > 150: + supplemental_text_pp = supplemental_text[0:150] + " ... " + else: + supplemental_text_pp = supplemental_text + + print(f"update: using lookup - {company_name} - found secondary source article " + f"(extract displayed) - ", supplemental_text_pp) + + # finally, try to get the company founding date from the supplemental text + new_response = model.function_call(supplemental_text,params=["company founding date"]) + + print("\nupdate: reviewed second source article - ", new_response["llm_response"]) + + if "company_founding_date" in new_response["llm_response"]: + company_founding_date = new_response["llm_response"]["company_founding_date"] + if company_founding_date: + print("update: success - found the answer - ", company_founding_date) + + return company_founding_date + + +if __name__ == "__main__": + + founding_date = not_found_then_triage_lookup() + + diff --git a/solutions/slim_agents/running_slim_custom_tests.py b/solutions/slim_agents/running_slim_custom_tests.py new file mode 100644 index 0000000..eecd3d5 --- /dev/null +++ b/solutions/slim_agents/running_slim_custom_tests.py @@ -0,0 +1,50 @@ + +""" This example shows how to use a custom test script in conjunction with a slim model tool test run. """ + +from llmware.models import ModelCatalog + +# custom test should be a json / python list of dictionaries with minimally a 'context' key for most models. +# some models require a "query" key +# optional "answer" key will be used in display if provided +# to check the existing tests, look @: /llmware_data/model_repo/{model_name}/config.json file + +custom_test = [ + + {"context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data and a " + "major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, or " + "0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy Nasdaq " + "Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in August, " + "the Labor Department said. Economists polled by Dow Jones expected 273,000 jobs. However, " + "wages rose less than expected last month. Stocks posted a stunning turnaround on Friday, " + "after initially falling on the stronger-than-expected jobs report. "}, + + {"context": "The Nasdaq and the S&P 500 slid by 0.8% during their lowest points in the day. Some noted it " + "could be the softer wage number in the jobs report that made investors confirm their bearish " + "stance. Others noted the pullback in yields from the day’s highs. 'We’re seeing a little " + "bit of a give back in yields from where we were around 4.8%. 'We’ve had a lot of weakness " + "in the market in recent weeks, and potentially some oversold conditions.'"}, + + {"context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following third " + "quarter earnings that plunged. The Finnish telecommunications giant said that it will reduce " + "its cost base and increase operation efficiency to “address the challenging market environment. " + "The substantial layoffs come after Nokia reported third-quarter net sales declined 20% year-on-year " + "to 4.98 billion euros. Profit over the period plunged by 69% year-on-year to 133 million euros."}, + + {"context": "Moody’s Investors Service lowered its ratings outlook on the United States’ government to " + "negative from stable, pointing to rising risks to the nation’s fiscal strength."}, + + {"context": "'The world's most advanced AI models are coming together with the world's most universal " + "user interface - natural language - to create a new era of computing,' said Satya Nadella, " + "chairman and chief executive officer of Microsoft. 'Across the Microsoft Cloud, we are the " + "platform of choice to help customers get the most value out of their digital spend and " + "innovate for this next generation of AI.' 'Focused execution by our sales teams and " + "partners in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive vice " + "president and chief financial officer of Microsoft."} + ] + +# pass optional custom_test_script parameter +# if no custom_test_script passed, then the default test set will be used automatically + +ModelCatalog().tool_test_run("slim-sentiment-tool", custom_test_script=custom_test) + diff --git a/solutions/slim_agents/sentiment-analysis.py b/solutions/slim_agents/sentiment-analysis.py new file mode 100644 index 0000000..7ea3e50 --- /dev/null +++ b/solutions/slim_agents/sentiment-analysis.py @@ -0,0 +1,98 @@ + +""" Sentiment Analysis example - shows how to use the slim-sentiment-tool. In this example, we will: + + 1. Review several summary earnings transcripts, looking to evaluate the overall sentiment as + 'positive', 'negative', or 'neutral' + + 2. Evaluate a single transcript, and apply if...then based on the result and confidence level. + + 3. Run through a list of earnings transcripts with journaling activated to display the multi-step + process on the screen. +""" + +from llmware.agents import LLMfx + +earnings_transcripts = [ + "This is one of the best quarters we can remember for the industrial sector with significant growth across the " + "board in new order volume, as well as price increases in excess of inflation. We continue to see very strong " + "demand, especially in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would be " + "accumulating more stock on any dips. ", + + "Not the worst results, but overall we view as negative signals on the direction of the economy, and the likely " + "short-term trajectory for the telecom sector, and especially larger market leaders, including AT&T, Comcast, and" + "Deutsche Telekom.", + + "This quarter was a disaster for Tesla, with falling order volume, increased costs and supply, and negative " + "guidance for future growth forecasts in 2024 and beyond.", + + "On balance, this was an average result, with earnings in line with expectations and no big surprises to either " + "the positive or the negative." + ] + + +def get_one_sentiment_classification(text): + + """This example shows a basic use to get a sentiment classification and use the output programmatically. """ + + # simple basic use to get the sentiment on a single piece of text + agent = LLMfx(verbose=True) + agent.load_tool("sentiment") + sentiment = agent.sentiment(text) + + # look at the output + print("sentiment: ", sentiment) + for keys, values in sentiment.items(): + print(f"{keys}-{values}") + + # two key attributes of the sentiment output dictionary + sentiment_value = sentiment["llm_response"]["sentiment"] + confidence_level = sentiment["confidence_score"] + + # use the sentiment classification as a 'if...then' decision point in a process + if "positive" in sentiment_value: + print("sentiment is positive .... will take 'positive' analysis path ...", sentiment_value) + + if "positive" in sentiment_value and confidence_level > 0.8: + print("sentiment is positive with high confidence ... ", sentiment_value, confidence_level) + + return sentiment + + +def review_batch_earning_transcripts(): + + """ This example highlights how to review multiple earnings transcripts and iterate through a batch + using the load_work mechanism. """ + + agent = LLMfx() + agent.load_tool("sentiment") + + # iterating through a larger list of samples + # note: load_work method is a flexible input mechanism - pass a string, list, dictionary or combination, and + # it will 'package' as iterable units of processing work for the agent + + agent.load_work(earnings_transcripts) + + while True: + output = agent.sentiment() + # print("update: test - output - ", output) + if not agent.increment_work_iteration(): + break + + response_output = agent.response_list + + agent.clear_work() + agent.clear_state() + + return response_output + + +if __name__ == "__main__": + + # first - quick illustration of getting a sentiment classification + # and using in an "if...then" + sentiment = get_one_sentiment_classification(earnings_transcripts[0]) + + # second - iterate thru a batch of transcripts and apply a sentiment classification + # response_output = review_batch_earning_transcripts() + + diff --git a/solutions/slim_agents/slims-getting-started.py b/solutions/slim_agents/slims-getting-started.py new file mode 100644 index 0000000..f361d74 --- /dev/null +++ b/solutions/slim_agents/slims-getting-started.py @@ -0,0 +1,159 @@ + +""" Getting Started with SLIM classifier function calling models - this script demonstrates seven + mini examples to get started using SLIMs: + + 1. Discover list of SLIM models. + 2. 'Hello World' first inference with SLIM model. + 3. Models vs. Tools + 4. Download and cache the SLIM tools. + 5. Run automated tests to confirm installation and demonstrate output. + 6. Using with LLMWare Prompts. + 7. Using the new LLMfx class. + +""" + +from llmware.models import ModelCatalog +from llmware.agents import LLMfx +from llmware.prompts import Prompt + + +def step1_discover_and_load_slim_models(): + + """ Discover a list of SLIM tools in the Model Catalog """ + + tools = ModelCatalog().list_llm_tools() + tool_map = ModelCatalog().get_llm_fx_mapping() + + print("\nList of SLIM model tools in the ModelCatalog\n") + + for i, tool in enumerate(tools): + model_card = ModelCatalog().lookup_model_card(tool_map[tool]) + print("update: step1 - slim tools: ", i, tool, model_card) + + return 0 + + +def step2_hello_world_slim(): + + """ SLIM models can be identified in the ModelCatalog like any llmware model. Instead of using + inference method, SLIM models are used with the function_call method that prepares a special prompt + instruction, and takes optional parameters. """ + + print("\n'Hello World' Inference Using SLIM Function call\n") + + # load like any other model anytime + model = ModelCatalog().load_model("slim-ner-tool") + response = model.function_call("Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s.") + + print("update: step2 - response: ", response) + print("update: step2 - usage: ", response["usage"]) + + return 0 + + +def step3_models_versus_tools(): + + """ All SLIM models are delivered in two different packages - as a traditional 'model' and as a + quantized 'tool.' In most scenarios, the tool is intended to be used for fast inference. """ + + print("\nSLIMs come packaged as 'models' (pytorch) and 'tools' (gguf)\n") + + model = ModelCatalog().load_model("llmware/slim-ner") + response = model.function_call("Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s.") + + print("update: step3 - response: ", response) + print("update: step3 - usage: ", response["usage"]) + + return 0 + + +def step4_load_and_cache_slim_tools(): + + """ To cache the SLIM toolkit locally, use .get_llm_toolkit. If you prefer to select specific tools, + then you can pass a tool_list in the method call as shown below. """ + + # get all tools + ModelCatalog().get_llm_toolkit() + + # select specific tools + ModelCatalog().get_llm_toolkit(tool_list=["sentiment", "ner"]) + + return 0 + + +def step5_run_automated_tests(): + + """ Each of these one line commands will locally cache the model and then run a series of tests using + the model to demonstrate its use and confirm that installation locally was successfully. """ + + # running automated tests - see the tools in action + + tools= ["slim-extract-tool", "slim-xsum-tool", "slim-summary-tool", "slim-boolean-tool", + "slim-sentiment-tool" , "slim-topics-tool", "slim-ner-tool", "slim-ratings-tool", + "slim-emotions-tool", "slim-intent-tool", "slim-tags-tool", "slim-sql-tool", + "slim-category-tool", "slim-nli-tool", "slim-sa-ner-tool", "slim-tags-3b-tool"] + + # run tests for one tool + ModelCatalog().tool_test_run("slim-sentiment-tool") + + # run tests for a bunch of tools + for tool in tools: + # excluding sentiment, since ran above as separate test + if tool != "slim-sentiment-tool": + ModelCatalog().tool_test_run(tool) + + return 0 + + +def step6_simple_use_case(): + + """ This illustrates how to run a basic function call inference on a SLIM model used in conjunction with + a LLMWare prompt. """ + + text = ("This is Melinda Wyngardt from Silvertech Ventures. We are extremely unhappy with the delays in closing " + "the loan and are considering whether to cancel and back out of the deal.") + + tags_model = ModelCatalog().load_model("slim-tags-tool") + response = tags_model.function_call(text,get_logits=True) + print("update: step6 - 'tags' response: ", response) + + intent_model = ModelCatalog().load_model("slim-intent-tool") + response2 = intent_model.function_call(text) + print("update: step6 - 'intent' response: ", response2) + + prompter = Prompt().load_model("llmware/bling-tiny-llama-v0") + output = prompter.prompt_main("What is the name of the company?", context=text) + print("update: step6 - 'question/answer' response: ", output) + + return 0 + + +def step7_introducing_llm_fx_class(): + + """ In addition to using SLIM models to 'supplement' primary LLM calls, SLIMs can be orchestrated in a + multi-step, multi-model workflow using the high-level LLMfx() - more examples on LLMfx() are in the next + main example 'agent-llmfx-getting-started.py' """ + + # shift verbose to True to see step-by-step processing on the screen + agent = LLMfx(verbose=False) + agent.load_tool("sentiment") + + text = "That is the worst thing that I have ever heard." + response = agent.exec_function_call("sentiment", text) + + print("update: step 7 - response - ", response) + + return 0 + + +if __name__ == "__main__": + + step1_discover_and_load_slim_models() + step2_hello_world_slim() + step3_models_versus_tools() + step4_load_and_cache_slim_tools() + step5_run_automated_tests() + step6_simple_use_case() + step7_introducing_llm_fx_class() + + diff --git a/solutions/slim_agents/text-2-sql-query-db.py b/solutions/slim_agents/text-2-sql-query-db.py new file mode 100644 index 0000000..56767ac --- /dev/null +++ b/solutions/slim_agents/text-2-sql-query-db.py @@ -0,0 +1,182 @@ + +""" This example shows an end-to-end recipe for querying SQL database using only natural language. + + The example shows the following steps: + + 1. Loading "slim-sql-tool" and running initial tests to confirm installation. + 2. Generating a SQL table from a sample CSV file included with the slim-sql-tool install. + 3. Asking basic natural language questions: + A. Looks up the table schema + B. Packages the table schema with query + C. Runs inference to convert text into SQL + D. Queries the database with the generated SQL + E. Returns result + 4. 'Two-step' query (starting on line 133) in which a customer name is pulled from a text using NER, and then + the name is 'dynamically' added to a natural language string, which is then converted using text-to-sql + and querying the database. + 5. All work performed on an integrated 'llmware-sqlite-experimental.db' that can be deleted safely anytime + as part of experimentation lifecycle. + +""" + +import os + +from llmware.agents import SQLTables, LLMfx +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + + +def load_slim_sql_tool(): + + """ First step is to install the slim-sql-tool locally """ + + # to cache locally the slim-sql-tool with config and test files + ModelCatalog().get_llm_toolkit(["sql"]) + + # to run tests to confirm correct installation and see the model in action + # note: the test results will include some minor errors - useful to learn how to sharpen prompts + ModelCatalog().tool_test_run("slim-sql-tool") + + return 0 + + +def hello_world_text_2_sql(): + + """ Illustrates a 'hello world' text-2-sql inference as part of an agent process. """ + sample_table_schema = "CREATE TABLE customer_info (customer_name text, account_number integer, annual_spend integer)" + query = "What are the names of all customers with annual spend greater than $1000?" + + agent = LLMfx(verbose=True) + response = agent.sql(query, sample_table_schema) + + print("update: text-2-sql response - ", response) + + return 0 + + +def build_table(fp, fn, table_name): + + """ This is the key method for taking a CSV file from a folder_path (fp), a proposed new table_name, + and creating a new table directly from the CSV. Note: this is useful for rapid prototyping and + experimentation - but should not be used for any serious production purpose. """ + + sql_db = SQLTables(experimental=True) + x = sql_db.create_new_table_from_csv(fp,fn,table_name=table_name) + print("update: successfully created new db table") + + return 1 + + +def delete_table(table_name): + + """ Start fresh in testing - delete table in experimental local SQLite DB """ + sql_db = SQLTables(experimental=True) + sql_db.delete_table(table_name,confirm_delete=True) + + return True + + +def delete_db(): + + """ Start fresh in testing - deletes SQLite DB and starts over. """ + + sql_db = SQLTables(experimental=True) + sql_db.delete_experimental_db(confirm_delete=True) + + return True + + +def sql_e2e_test_script(table_name="customers1",create_new_table=False): + + """ This is the end-to-end execution script. """ + + # create table if needed to set up + if create_new_table: + + sql_tool_repo_path = os.path.join(LLMWareConfig().get_model_repo_path(), "slim-sql-tool") + + if not os.path.exists(sql_tool_repo_path): + ModelCatalog().load_model("llmware/slim-sql-tool") + + files = os.listdir(sql_tool_repo_path) + + csv_file = "customer_table.csv" + + if csv_file in files: + build_table(sql_tool_repo_path, csv_file, table_name) + else: + print("something has gone wrong - could not find customer_table.csv with slim-sql-tool file package") + + # query starts here + agent = LLMfx() + agent.load_tool("sql") + + # Example 1 - direct query + + query_list = ["Which customers are vip customers?", + "What is the highest annual spend of any customer?", + "Which customer has account number 1234953", + "Which customer has the lowest annual spend?", + "Is Susan Soinsin a vip customer?"] + + for i, query in enumerate(query_list): + + # this method is doing all of the work + # -- looks up the table schema in the db using the table_name + # -- packages the text-2-sql query prompt + # -- executes sql method to convert the prompt into a sql query + # -- attempts to execute the sql query on the db + # -- returns the db results as 'research' output + + response = agent.query_db(query, table=table_name) + + # Example 2 - use in a chain of inferences + + text = ("This is Susan Soinsin calling - I am really upset about the poor customer service, " + "and would like to cancel my service.") + + agent.load_tool("ner") + response = agent.ner(text=text) + customer_name = "No Customer" + + # please note: this is just a demo recipe - any real life scenario would require significant preprocessing + # and error checking. :) + + if "llm_response" in response: + if "people" in response["llm_response"]: + people = response["llm_response"]["people"] + if len(people) > 0: + customer_name = people[0] + + print("ner response: ", customer_name, response) + + # e.g., name = "Susan Soinsin" + + query = f"Is {customer_name} a vip customer?" + + print("query: ", query) + + response = agent.query_db(query, table=table_name) + + print("response: ", response) + + for x in range(0,len(agent.research_list)): + print("research: ", x, agent.research_list[x]) + + return 0 + + +if __name__ == "__main__": + + # first - load and test the tools + load_slim_sql_tool() + + # second - 'hello world' demo of using text2sql model + hello_world_text_2_sql() + + # second - run an end-to-end test + sql_e2e_test_script(table_name="customer1",create_new_table=True) + + # third - delete and start fresh for further testing + delete_table("customer1") + diff --git a/solutions/slim_agents/text2sql-end-to-end-2.py b/solutions/slim_agents/text2sql-end-to-end-2.py new file mode 100644 index 0000000..6ce7d2d --- /dev/null +++ b/solutions/slim_agents/text2sql-end-to-end-2.py @@ -0,0 +1,113 @@ + +""" This example shows an end-to-end recipe for querying SQL database using only natural language. + + The example shows the following steps: + + 1. Loading "slim-sql-tool" and running initial tests to confirm installation. + 2. Generating a SQL table from a sample CSV file included with the slim-sql-tool install. + 3. Asking basic natural language questions: + A. Looks up the table schema + B. Packages the table schema with query + C. Runs inference to convert text into SQL + D. Queries the database with the generated SQL + E. Returns result + 3. All work performed on an integrated 'llmware-sqlite-experimental.db' that can be deleted safely anytime + as part of experimentation lifecycle. + + UPDATE: please see also the related example in Use_Cases/agent_with_custom_tables.py, which illustrates a more + generalized version of this script running on Postgres. + +""" + +import os + +from llmware.agents import SQLTables, LLMfx +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + + +def sql_e2e_test_script(table_name="customers1",create_new_table=False): + + """ This is the end-to-end execution script. """ + + # create table if needed to set up + if create_new_table: + + # looks to pull sample csv 'customer_table.csv' from slim-sql-tool model package files + sql_tool_repo_path = os.path.join(LLMWareConfig().get_model_repo_path(), "slim-sql-tool") + + if not os.path.exists(sql_tool_repo_path): + ModelCatalog().load_model("llmware/slim-sql-tool") + + files = os.listdir(sql_tool_repo_path) + csv_file = "customer_table.csv" + + if csv_file in files: + + # to create a testing table from a csv + sql_db = SQLTables(experimental=True) + sql_db.create_new_table_from_csv(sql_tool_repo_path, csv_file, table_name=table_name) + # end - creating table + + print("update: successfully created new db table") + else: + print("something has gone wrong - could not find customer_table.csv inside the slim-sql-tool file package") + + # query starts here + agent = LLMfx() + agent.load_tool("sql", sample=False, get_logits=True, temperature=0.0) + + # Pass direct queries to the DB + + query_list = ["Which customers have vip customer status of yes?", + "What is the highest annual spend of any customer?", + "Which customer has account number 1234953", + "Which customer has the lowest annual spend?", + "Is Susan Soinsin a vip customer?"] + + for i, query in enumerate(query_list): + + # query_db method is doing all of the work + # -- looks up the table schema in the db using the table_name + # -- packages the text-2-sql query prompt + # -- executes sql method to convert the prompt into a sql query + # -- attempts to execute the sql query on the db + # -- returns the db results as 'research' output + + response = agent.query_db(query, table=table_name) + + for x in range(0,len(agent.research_list)): + print("research: ", x, agent.research_list[x]) + + return 0 + +def delete_table(table_name): + + """ Start fresh in testing - delete table in experimental local SQLite DB """ + + sql_db = SQLTables(experimental=True) + sql_db.delete_table(table_name, confirm_delete=True) + + return True + + +def delete_db(): + + """ Start fresh in testing - deletes SQLite DB and starts over. """ + + sql_db = SQLTables(experimental=True) + sql_db.delete_experimental_db(confirm_delete=True) + + return True + + +if __name__ == "__main__": + + ModelCatalog().get_llm_toolkit(tool_list=["sql"]) + + # run an end-to-end test + sql_e2e_test_script(table_name="customer1",create_new_table=True) + + # third - delete and start fresh for further testing + delete_table("customer1") + diff --git a/solutions/slim_agents/text2sql-getting-started-1.py b/solutions/slim_agents/text2sql-getting-started-1.py new file mode 100644 index 0000000..635640c --- /dev/null +++ b/solutions/slim_agents/text2sql-getting-started-1.py @@ -0,0 +1,53 @@ + +""" This 'getting started' example shows the basics of how to start using text2sql model: + + 1. Loading "slim-sql-tool" and running initial tests to confirm installation. + + 2. 'Hello World' demonstration of how to 'package' a text2sql prompt combining a + natural language query with a SQL table schema and run a basic inference to generate SQL output + +""" + + +from llmware.agents import LLMfx +from llmware.models import ModelCatalog + + +def load_slim_sql_tool(): + + """ First step is to install the slim-sql-tool locally """ + + # to cache locally the slim-sql-tool with config and test files + ModelCatalog().get_llm_toolkit(["sql"]) + + # to run tests to confirm correct installation and see the model in action + # note: the test results will include some minor errors - useful to learn how to sharpen prompts + ModelCatalog().tool_test_run("slim-sql-tool") + + return 0 + + +def hello_world_text_2_sql(): + + """ Illustrates a 'hello world' text-2-sql inference as part of an agent process. """ + + sample_table_schema = "CREATE TABLE customer_info (customer_name text, account_number integer, annual_spend integer)" + + query = "What are the names of all customers with annual spend greater than $1000?" + + agent = LLMfx(verbose=True) + response = agent.sql(query, sample_table_schema) + + print("update: text-2-sql response - ", response) + + return response + + +if __name__ == "__main__": + + # first - load and test the tools + load_slim_sql_tool() + + # second - 'hello world' demo of using text2sql model + hello_world_text_2_sql() + diff --git a/solutions/slim_agents/text2sql-multistep-example-3.py b/solutions/slim_agents/text2sql-multistep-example-3.py new file mode 100644 index 0000000..0a25478 --- /dev/null +++ b/solutions/slim_agents/text2sql-multistep-example-3.py @@ -0,0 +1,115 @@ + + +""" This example shows a multi-step SQL query use case - this is an 'innovation scenario' and should be viewed +as a good starting recipe for building your own more complex workflows involving text2sql queries. + + The example shows the following steps: + + 1. Generating a SQL table from a sample CSV file included with the slim-sql-tool install. + 2. 'Two-step' query (starting on line 133) in which a customer name is pulled from a text using NER, and then + the name is 'dynamically' added to a natural language string, which is then converted using text-to-sql + and querying the database. + 3. All work performed on an integrated 'llmware-sqlite-experimental.db' that can be deleted safely anytime + as part of experimentation lifecycle. + +""" + +import os + +from llmware.agents import SQLTables, LLMfx +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + +llmware_path = LLMWareConfig().get_llmware_path() + + +def sql_two_step_query_example(table_name="customers1",create_new_table=False): + + """ This is the end-to-end execution script. """ + + # create table if needed to set up + if create_new_table: + + sql_tool_repo_path = os.path.join(LLMWareConfig().get_model_repo_path(), "slim-sql-tool") + + if not os.path.exists(sql_tool_repo_path): + ModelCatalog().load_model("llmware/slim-sql-tool") + + files = os.listdir(sql_tool_repo_path) + + csv_file = "customer_table.csv" + + if csv_file in files: + sql_db = SQLTables(experimental=True) + sql_db.create_new_table_from_csv(sql_tool_repo_path, csv_file, table_name=table_name) + print("update: successfully created new db table") + + else: + print("something has gone wrong - could not find customer_table.csv with slim-sql-tool file package") + + # query starts here + agent = LLMfx() + agent.load_tool("sql") + agent.load_tool("ner") + + # Multi-step example - extract NER -> create natural language query -> convert SQL -> lookup + + text = ("This is Susan Soinsin calling - I am really upset about the poor customer service, " + "and would like to cancel my service.") + + # Step 1 - extract the customer name using NER + response = agent.ner(text=text) + customer_name = "No Customer" + + # please note: this is just a demo recipe - any real life scenario would require significant preprocessing + # and error checking. :) + + if "llm_response" in response: + if "people" in response["llm_response"]: + people = response["llm_response"]["people"] + if len(people) > 0: + customer_name = people[0] + + print("update: ner response - identified the following people names - ", customer_name, response) + + # Step 2 - use the customer name found in the NER analysis to construct a natural language query + query = f"Is {customer_name} a vip customer?" + + print("update: dynamically created query: ", query) + + response = agent.query_db(query, table=table_name) + + print("update: response: ", response) + + for x in range(0,len(agent.research_list)): + print("research: ", x, agent.research_list[x]) + + return 0 + +def delete_table(table_name): + + """ Start fresh in testing - delete table in experimental local SQLite DB """ + sql_db = SQLTables(experimental=True) + sql_db.delete_table(table_name,confirm_delete=True) + + return True + + +def delete_db(): + + """ Start fresh in testing - deletes SQLite DB and starts over. """ + + sql_db = SQLTables(experimental=True) + sql_db.delete_experimental_db(confirm_delete=True) + + return True + + +if __name__ == "__main__": + + # second - run an end-to-end test + sql_two_step_query_example (table_name="customer1",create_new_table=True) + + # third - delete and start fresh for further testing + delete_table("customer1") + diff --git a/solutions/slim_agents/using-slim-output-values.py b/solutions/slim_agents/using-slim-output-values.py new file mode 100644 index 0000000..beecca6 --- /dev/null +++ b/solutions/slim_agents/using-slim-output-values.py @@ -0,0 +1,92 @@ + +""" This example demonstrates the capabilities to use SLIM output values for programmatic evaluation, specifically + SLIM models that generate 'classification' oriented outputs. + + One of the exciting features of the SLIM models is the ability to generate natural language directly, rather than + simply 'slotting' an answer into a predefined category - as a result, we believe that the SLIM models can + generalize better, as the model has the ability to explicitly draw upon the objective in generating a + response. + + As a result, if you apply SLIM models to out-of-domain content, it is possible (even likely) that you may see a + different range of values than those outlined below + + The following SLIM models have outputs that tend to be 'classifiers' or labelled 'categories' of specific values: + + 1. sentiment - range of 3 values - positive, negative, neutral + + 2. ratings - range of 5 values - 1, 2, 3, 4, 5 ('degree' of sentiment) + + 3. nli - range of 3 values - supports, contradicts, neutral + + 4. emotions - ~35 emotion values - "afraid", "anger", "angry", "annoyed", "anticipating", "anxious", + "apprehensive", "ashamed", "caring", "confident", "content", + "devastated", "disappointed", "disgusted", "embarrassed", + "excited", "faithful", "fear", "furious", "grateful", "guilty", + "hopeful", "impressed", "jealous", "joy", "joyful", "lonely", + "love", "nostalgic", "prepared", "proud", "sad", "sadness", + "sentimental", "surprise", "surprised", "terrified", "trusting" + + slim-category was trained on a diverse range of business, financial and general news documents with the goal of + defining the category or larger topic associated with a particular text + + 5. category - ~27 category values - "analyst", "announcements", "bonds", "business", "central bank", + "commentary", "commodities", "currencies", "dividend", "earnings", + "energy", "entertainment", "financials", "health", + "human resources", "legal and regulation", "macroeconomics", + "markets", "mergers and acquisitions", "opinion", "politics", + "public markets", "science", "sports", "stocks", "tech", "world" + + slim-intent was trained with wide range of materials from customer service and dialogs with a focus on trying to + classify the intent of the customer's request + + 6. intent - ~14 values - "account", "cancel", "complaint", "customer service", "delivery", + "feedback", "invoice", "new account", "order", "payments", + "refund", "shipping", "subscription", "terminate" + + 7. topics - Generative Topic - the topics model was trained primarily on complex financial and + legal documents, but in our testing, the model generalizes very + well to almost any text - and will be 'generative' in providing + essentially a 1-2 word 'summary' of the text. + + The following models are generally 'extractive' in that the output values will have a wide spectrum of potential + values, based on the subject text: + + 8. slim-extract + + 9. slim-ner + + 10. slim-tags + + 11. slim-tags-3b + + 12. slim-summary + + 13. slim-boolean + + 14. slim-xsum + + 15. sli-sql + + 16. slim-sentiment-ner + + """ + + +from llmware.models import ModelCatalog + +models = ModelCatalog().list_function_call_models() + +for i, model_card in enumerate(models): + + model_name = model_card["model_name"] + + # to view the "function call primary keys" for a selected model + keys = ModelCatalog().fc_primary_keys(model_name) + + # to view the expected range of output values + values = ModelCatalog().fc_output_values(model_name) + + print("\nmodel_name: ", model_name) + print("primary keys/parameters: ", keys) + print("target output values: ", values) + diff --git a/solutions/slim_agents/using-slim-q-gen.py b/solutions/slim_agents/using-slim-q-gen.py new file mode 100644 index 0000000..d088983 --- /dev/null +++ b/solutions/slim_agents/using-slim-q-gen.py @@ -0,0 +1,137 @@ + +""" This example shows how to use the slim-q-gen models to automatically generate a question based on a +context passage. + + There are two 'q-gen' models - (a) tiny-llama base (1.1b), and (b) phi-3 base (3.8b) + + Both models work the same way with tiny-llama a little faster, and phi-3 a little higher quality. + + The models come packaged both as pytorch and gguf - for most inference use cases, we would recommend + the gguf versions which are considerably faster. + + We would recommend experimenting with the temperature settings to optimize varied and + creative question generations. + + Automated question generation has several use cases, including: + + -- quiz test question generation for education, enablement, self-training or testing + -- search retrieval tagging to add top questions to the search index (both text and semantic) + -- agent-oriented scenarios in which one model 'asks' the question, and another model 'answers' it + + models in catalog: + + -- "slim-q-gen-phi-3" + -- "slim-q-gen-phi-3-tool" + -- "slim-q-gen-tiny" + -- "slim-q-gen-tiny-tool" + + """ + +from llmware.models import ModelCatalog + + +def hello_world_test(source_passage, q_model="slim-q-gen-tiny-tool", number_of_tries=10, question_type="question", + temperature=0.5): + + """ Shows a basic example of generating questions from a text passage, running a number of inferences, + and then keeping only the unique questions generated. + + -- source_passage = text passage + -- number_of_tries = integer number of times to call the model to generate a question + -- question_type = "question" | "boolean" | "multiple choice" + + """ + + # recommend using temperature of 0.2 - 0.8 - for multiple choice, use lower end of the range + q_model = ModelCatalog().load_model(q_model, sample=True, temperature=temperature) + + questions = [] + + for x in range(0, number_of_tries): + + response = q_model.function_call(source_passage, params=[question_type], get_logits=False) + + # expect response in the form of: "llm_response": {"question": ["generated question?"] } + + if response: + if "llm_response" in response: + if "question" in response["llm_response"]: + new_q = response["llm_response"]["question"] + + # keep only new questions + if new_q not in questions: + questions.append(new_q) + + print(f"inference {x} - response: {response['llm_response']}") + + print(f"\nDe-duped list of questions created\n") + for i, question in enumerate(questions): + + print(f"new generated questions: {i} - {question}") + + return questions + + +def ask_and_answer_game(source_passage, q_model="slim-q-gen-tiny-tool", number_of_tries=10, question_type="question", + temperature=0.5): + + """ Shows a simple two model game of using q-gen model to generate a question, and then a second model + to answer the question generated. """ + + # this is the model that will generate the 'question' + q_model = ModelCatalog().load_model(q_model, sample=True, temperature=temperature) + + # this will be the model used to 'answer' the question + answer_model = ModelCatalog().load_model("bling-phi-3-gguf") + + questions = [] + + print(f"\nGenerating a set of questions automatically from the source passage.\n") + + for x in range(0,number_of_tries): + + response = q_model.function_call(source_passage, params=[question_type], get_logits=False) + + if response: + if "llm_response" in response: + if "question" in response["llm_response"]: + new_q = response["llm_response"]["question"] + + # only keep new questions + if new_q and new_q not in questions: + questions.append(new_q) + + print(f"inference - {x} - response: {response}") + + print("\nAnswering the generated questions\n") + for i, question in enumerate(questions): + + print(f"\nquestion: {i} - {question}") + if isinstance(question, list) and len(question) > 0: + response = answer_model.inference(question[0], add_context=test_passage) + print(f"response: ", response["llm_response"]) + + return True + + +if __name__ == "__main__": + + # test passage pulled from CNBC news story on Tuesday, May 28, 2024 + test_passage = ("OpenAI said Tuesday it has established a new committee to make recommendations to the " + "company’s board about safety and security, weeks after dissolving a team focused on AI safety. " + "In a blog post, OpenAI said the new committee would be led by CEO Sam Altman as well as " + "Bret Taylor, the company’s board chair, and board member Nicole Seligman. The announcement " + "follows the high-profile exit this month of an OpenAI executive focused on safety, " + "Jan Leike. Leike resigned from OpenAI leveling criticisms that the company had " + "under-invested in AI safety work and that tensions with OpenAI’s leadership had " + "reached a breaking point.") + + # first example + hello_world_test(test_passage,q_model="slim-q-gen-tiny-tool",number_of_tries=10, + question_type="question", + temperature=0.5) + + # second example + ask_and_answer_game(test_passage,q_model="slim-q-gen-phi-3-tool", number_of_tries=10, + question_type="question", + temperature=0.5) diff --git a/solutions/slim_agents/using-slim-qa-gen.py b/solutions/slim_agents/using-slim-qa-gen.py new file mode 100644 index 0000000..bba81b9 --- /dev/null +++ b/solutions/slim_agents/using-slim-qa-gen.py @@ -0,0 +1,530 @@ + +""" This example shows how to use the slim-qa-gen models to automatically generate a question and answer based on a +context passage. + + There are two 'qa-gen' models - (a) tiny-llama base (1.1b), and (b) phi-3 base (3.8b) + + Both models work the same way with tiny-llama a little faster, and phi-3 a little higher quality. + + The models come packaged both as pytorch and gguf - for most inference use cases, we would recommend + the gguf versions which are considerably faster. + + This example uses an earnings_release test set that was also included in using_slim_extract_model.py, primarily + because it generates an interesting set of questions and answers. Feel free to substitute your own source + test sets. + + In the example, we will show how to take the generated question-answer pairs and create a mini self-supervised + instruct dataset. + + models in catalog: + + -- "slim-qa-gen-phi-3" + -- "slim-qa-gen-phi-3-tool" + -- "slim-qa-gen-tiny" + -- "slim-qa-gen-tiny-tool" + + """ + +import json +import os +import random + +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs +from llmware.configs import LLMWareConfig + + +def earning_releases_test_set(): + + earnings_releases = [ + + {"context": "Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker " + "issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. " + "Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: " + "Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected " + "Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. " + "Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, " + "in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of " + "design software startup Figma after U.K. regulators found competitive concerns. The company paid " + "Figma a $1 billion termination fee."}, + + { + "context": "Dick’s Sporting Goods raised its dividend by 10% on Thursday as the company posted its largest sales " + "quarter in its history and projected another year of growth. The company’s shares jumped more than " + "15% in intraday trading. CEO Lauren Hobart said on an earnings call Thursday that Dick’s sales " + "growth came from bigger tickets — either higher prices or more expensive items — as its transactions " + "were flat. Many retailers benefited from a 53rd week in fiscal 2023, but Dick’s said it still broke " + "records during its fiscal fourth quarter even without those extra days. Here’s how the athletic " + "apparel retailer did compared with what Wall Street was anticipating, based on a survey of " + "analysts by LSEG, formerly known as Refinitiv: Earnings per share: $3.85 adjusted vs. $3.35 expected " + "Revenue: $3.88 billion vs. $3.80 billion expected The company’s reported net income for the three-month " + "period that ended Feb. 3 was $296 million, or $3.57 per share, compared with $236 million, or $2.60 a " + "share, a year earlier. Excluding one-time items related to impairment charges and inventory write-offs, " + "Dick’s reported earnings per share of $3.85. Sales rose to $3.88 billion, up about 8% from $3.60 billion " + "a year earlier. “With our industry-leading assortment and strong execution, we capped off the year " + "with an incredibly strong fourth quarter and holiday season,” Hobart said in a statement. “We are " + "guiding to another strong year in 2024. We plan to grow both our sales and earnings through " + "positive comps, higher merchandise margin and productivity gains,” she added. During the quarter, " + "same-store sales rose 2.8%, well ahead of the 0.8% lift that analysts had expected, according to " + "StreetAccount. “Growth in transactions” and market share gains drove the increase, said Executive " + "Chairman Ed Stack."}, + + {"context": "Comcast topped both revenue and profit estimates in the fourth quarter as it lost fewer broadband " + "subscribers than expected, and it raised its dividend 7%, the company said Thursday. " + "Here’s how Comcast performed, compared with estimates from analysts surveyed by LSEG, " + "formerly known as Refinitiv. Earnings per share: 84 cents adjusted vs. 79 cents expected " + "Revenue: $31.25 billion vs. $30.51 billion expected For the quarter ended Dec. 31, net " + "income rose 7.8% to $3.26 billion, or 81 cents a share, compared with $3.02 billion, or " + "70 cents a share, a year earlier. Revenue increased 2.3% compared with the prior-year period. " + "Adjusted earnings before interest, taxes, depreciation and amortization (EBITDA) was flat year " + "over year at about $8 billion. 'For the third consecutive year, we generated the highest revenue, " + "adjusted EBITDA and adjusted EPS in our company’s history', Comcast Chief Executive Officer Brian " + "Roberts said in a statement. 'We also reported the highest adjusted EBITDA on record at Theme Parks; " + "were the #1 studio in worldwide box office for the first time since 2015; and maintained Peacock’s " + "position as the fastest growing streamer in the U.S.'"}, + + {"context": "Dollar General forecast annual sales above Wall Street estimates on Thursday, banking on higher " + "demand from inflation-hit customers buying groceries and essentials from the discount retailer’s stores. " + "Shares of the company rose about 6% in early trading, after falling nearly 45% in 2023 on rising costs " + "and stiff competition from bigger retailers. But higher prices and borrowing costs have prompted " + "budget-conscious consumers to cook more meals at home, helping Dollar General record stronger " + "footfall at its outlets as shoppers hunt for lower-margin, needs-based goods, over pricier general " + "merchandise. “With customer traffic growth and market share gains during the quarter, we believe our " + "actions are resonating with customers,” CEO Todd Vasos said in a statement. Vasos’s strategy - to focus " + "on the basics, like more employee presence at stores, greater customer engagement and expanding " + "private-label brands - has helped stabilize Dollar General’s business. Over the last few quarters, " + "Dollar General and rival Dollar Tree have struggled with rising costs linked to their supply " + "chains, labor and raw materials, while facing tough competition from retailers like Walmart " + "and Chinese ecommerce platform Temu. Dollar Tree’s shares fell more than 15% on Wednesday, after it " + "forecast weak sales and profit for 2024 and laid out plans to shutter 970 of its Family Dollar " + "stores. “Dollar General has a much rosier outlook than Dollar Tree... Dollar Tree’s challenges " + "with Family Dollar were years in the making, while Dollar General has embarked on an aggressive " + "effort to add more frozen, refrigerated and fresh produce,” eMarketer senior analyst Zak Stambor said. " + "Dollar General forecast 2024 sales to grow between 6.0% and 6.7%, above analysts’ estimate of 4.4% " + "growth to $40.33 billion, according to LSEG data. It still sees annual per-share profit between " + "$6.80 and $7.55, compared with estimates of $7.55. Its fourth-quarter net sales of $9.86 billion " + "surpassed estimates of $9.78 billion. It also reported an estimate-beating profit of $1.83 per share."}, + + { + "context": "Shares of Zara owner Inditex hit record highs on Wednesday according to LSEG data, climbing over 6% during " + "intraday trading after the company announced its 2023 full-year results. As of 11:50 London time, shared " + "were just over 6% higher at 43.58 euros, or $47.69. Sales increased 10.4% to 35.9 billion euros for the " + "year, the company said, signaling this was a record high. Sales grew across all geographic regions and " + "across Inditex’s brands and were “very satisfactory,” both online and in store, the company said. A total of " + "5,692 stores were operational at the end of the year, Inditex said, adding it plans to expand further in " + "2024, including with Zara shops in Los Angeles and Las Vegas. The company also plans to open new distribution " + "centers in 2024 and 2025, as part of a major logistics expansion plan that will cost the company " + "investments of 900 million euros in both years. Net income also reached a fresh high after soaring " + "30.3% from 2022 to reach 5.4 billion euros last year. The company’s gross profit came in at 20.8 billion " + "euros, up 11.9% on the year. “Inditex’s performance in 2023 has been excellent. Our teams have been able to " + "take advantage of the opportunities to keep growing profitably. We are investing to drive future growth and " + "continue to offer an attractive remuneration to shareholders,” Inditex CEO Oscar García Maceiras said in a " + "statement. The Spanish clothing company owns a range of vastly popular brands including household name " + "Zara, as well as Pull & Bear, Bershka, Stradivarius, premium retailer Massimo Dutti and sports and the " + "athleisure-focused Oysho. Zara, including the Zara Home range, was the biggest contributor to sales in " + "2023, followed by Pull & Bear and Massimo Dutti, Inditex said Wednesday. The company also indicated " + "that 2024 was off to a strong start, with sales in constant currency up 11% over the Feb. 1 to March 11 " + "stretch, compared with the same period a year earlier."}, + + { + "context": "Oracle reported quarterly earnings on Monday that exceeded Wall Street’s expectations. Shares rose " + "13% in extended trading. Here’s how the company did in the fiscal third quarter ending Feb. 29, compared " + "to estimates by LSEG, formerly known as Refinitiv: Earnings per share: $1.41 adjusted vs. $1.38 expected " + "Revenue: $13.28 billion vs. $13.3 billion expected For the fiscal fourth quarter, Oracle said it expects " + "earnings of $1.62 to $1.66 per share. Analysts were expecting $1.64 in adjusted earnings per share, according " + "to LSEG. Revenue growth will be between 4% and 6% over sales of $13.8 billion a year ago. The midpoint of that " + "range would equal revenue of about $14.5 billion, while analysts were expecting a little more than $14.7 billion. " + "Oracle CEO Safra Catz said the company was committed to hitting previously stated goals of $65 billion in " + "sales by fiscal 2026. “Some of these goals might prove to be too conservative given our momentum,” Catz said. " + "Revenue rose 7% in the quarter from $12.4 billion a year earlier. Net income climbed 27% to $2.4 billion, " + "or 85 cents per share, from $1.9 billion, or 68 cents per share, a year ago. Oracle’s cloud services and " + "license support segment, its largest business, saw sales rise 12% to $9.96 billion, slightly beating " + "StreetAccount consensus expectations of $9.94 billion. The company attributed the rise to strong demand " + "for its artificial intelligence servers. Catz said the company added several “large new cloud " + "infrastructure” contracts during the quarter. The company’s cloud revenue, which is reported as part " + "of the cloud services unit, rose 25% year over year to $5.1 billion, Oracle said. “We signed several large " + "deals this quarter and we have many more in the pipeline,” Catz told investors on the earnings call. " + "Oracle Chairman Larry Ellison cited increased business from Microsoft on the earnings call. " + "“We’re building 20 data centers from Microsoft and Azure. They just ordered three more data centers " + "this week,” Ellison said. The company’s other units didn’t fare as well. Cloud license and on-premise sales " + "declined 3% to $1.26 billion, slightly beating StreetAccount’s forecast. Hardware revenue fell 7% to " + "$754 million, while sales in the company’s services division slid 5% to $1.31 billion, both falling short " + "of StreetAccount expectations. Prior to Monday’s report, Oracle shares were up 8.7% for the year, " + "slightly outperforming the S&P 500."}, + + { + "context": "Porsche on Tuesday warned that profitability will decline this year as it launches new models amid " + "tough economic conditions, but hiked its dividend on the back of a rise in 2023 operating profit. The German " + "luxury automaker said it expects an operating return on sales of between 15% and 17% in 2024, down from the " + "18% margin notched in 2023 and 2022. In the long term, the group targets an operating return on sales of more " + "than 20%. Explaining the more cautious profitability outlook, the company cited “the comprehensive renewal " + "of its product range in 2024, the global framework conditions, higher depreciations on capitalized " + "development costs and the continued investments in the brand and the Porsche ecosystem.” The company’s shares " + "were around 4.8% higher by early afternoon, having reversed opening losses of more than 2%. Porsche is " + "launching four new car ranges in 2024 in the form of the Panamera, Macan, Taycan and 911 model lines. " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China WATCH NOW VIDEO 03:17 " + "Porsche CFO: Expect significant growth in high-net-worth individuals in China “2024 is going to be a year of " + "product launches for Porsche – more so than any year in our history,” Chairman Oliver Blume said in a " + "statement. “We will be introducing a variety of exhilarating sports cars to the road, they will delight " + "our customers around the world. This will put the wind at our back for years to come.”"}, + + { + "context": "Lego on Tuesday reported its full-year 2023 results, saying it’s revenue grew by 2% throughout the year, " + "in line with expectations. The company made “very, very strong progress” and “grew comfortably” in the " + "U.S., its CEO Niels Christiansen told CNBC. The toy industry has been struggling to maintain " + "pandemic-era growth as inflation is putting pressure on demand and sales. In-store participation is greater " + "than prior to the pandemic, Lego CEO says In-store participation is greater than prior " + "to the pandemic, Lego CEO says The chief executive of Denmark’s Lego on Tuesday reflected on a tough year " + "for the world’s largest toymaker, and outlined the firm’s long-term plans to stay relevant and “cool with kids. ”" + "Lego said its 2023 revenue was 2% higher compared to the previous year, growing to 65.9 billion Danish krone " + "(around $9.65 billion). This was in line with expectations, Lego said in a statement. “It was a difficult year,” " + "Lego CEO Niels Christiansen told CNBC. However, he said the company had “managed to take quite a bit of " + "market share.” The Danish toymaker said operating profit declined slightly from 17.9 billion Danish krone " + "to 17.1 billion, noting that it had boosted spending on strategic initiatives designed to drive growth. " + "Net profit came in at 13.1 billion Danish krone in 2023, compared to 13.8 billion the previous year. " + "Consumer sales were up 4% despite slumping in China, Lego said, attributing the growth to increasing demand " + "in the U.S. and central and eastern Europe. It comes as the wider toy industry has been struggling to " + "maintain growth after booming during the coronavirus pandemic, when parents looked for new ways to " + "entertain their children and adults re-discovered childhood pastimes. Toy company Hasbro earlier this month " + "said its 2023 revenue fell by 15% compared to 2022 and that it expected to see a further decline this year."}, + + { + "context": "Adidas on Wednesday warned of a sales decline in its overstocked North American market in 2024, as the " + "German sportswear brand continues to sell off its remaining Yeezy inventory. Currency-neutral sales in " + "North America are expected to decline to a mid-single-digit rate in 2024, but are projected to notch " + "mid-single-digit growth worldwide despite persistent “macroeconomic challenges and geopolitical tensions,” " + "the company said. Adidas confirmed its 2023 operating profit came in at 268 million euros ($292.9 million) " + "on the back of flat currency-neutral sales, significantly above prior expectations as the company continues " + "to take a hit from the cessation of its line of Yeezy — footwear the retailer produced in a collaboration with " + "American rapper Ye, formerly known as Kanye West. For the fourth quarter, the company posted an operating " + "loss of 377 million euros. The board proposed a flat dividend of 0.70 euros per share. “Although by far not " + "good enough, 2023 ended better than what I had expected at the beginning of the year,” CEO Bjørn Gulden " + "said in a statement. “Despite losing a lot of Yeezy revenue and a very conservative sell-in strategy, " + "we managed to have flat revenues. We expected to have a substantial negative operating result, but " + "achieved an operating profit of €268 million.” Adidas was confirming preliminary results released in late " + "January, when it announced that it would not write off the majority of its Yeezy inventory and would instead " + "sell off the remaining shoes at cost. The sportswear giant was forced to axe the Yeezy line after terminating " + "its partnership with Ye over a string of anti-Semitic remarks that the rapper made in 2022. Adidas said the " + "discontinuation of Yeezy represented a drag of around 500 million euros in the year-on-year comparison " + "through 2023, though the sale of parts of the remaining inventory in the second and third quarter positively " + "impacted net sales by around 750 million euros. “With a very disciplined go-to-market and buying process, " + "we reduced our inventories by almost €1.5 billion. With the exception of the U.S., we now have healthy " + "inventories everywhere,” Gulden said. He added that the company is expecting some growth in the " + "first quarter of 2024 and a further pick-up in the second half of the year. “We still have a lot of work " + "to do, but I feel very confident we are on the right track. We will bring adidas back again. Give us some " + "time and we will again say – we got this!” he said. Adidas projected an operating profit of around " + "500 million euros in 2024, with unfavorable currency effects expected to “weigh significantly on the " + "company’s profitability” because of adverse impacts on both reported revenues and gross margin development." + "Adidas shares were flat by mid-morning on Wednesday. Mamta Valechha, equity research analyst at " + "Quilter Cheviot, said that, given that the headline numbers were already pre-released in January, the most " + "interesting aspect of Wednesday’s report was the “clear acceleration of the Adidas brand.”"}, + + { + "context": "Costco on Thursday missed Wall Street’s revenue expectations for its holiday quarter, despite reporting " + "year-over-year sales growth and strong e-commerce gains. Shares of the retailer fell about 4% in aftermarket " + "trading. The company’s stock had hit a 52-week high earlier in the day. Here’s what Costco reported for its " + "fiscal second quarter of 2024 compared with what Wall Street was expecting, based on a survey of analysts by " + "LSEG, formerly known as Refinitiv: Earnings per share: $3.92 vs. $3.62 expected Revenue: $58.44 billion vs. " + "$59.16 billion expected In the three-month period that ended Feb. 18, Costco’s net income rose to " + "$1.74 billion, or $3.92 per share, compared with $1.47 billion, or $3.30 per share, a year earlier. " + "Costco’s revenue for the quarter increased from $55.27 billion in the year-ago period. Comparable sales for " + "the company increased 5.6% year over year and 4.3% in the U.S. Excluding changes in gas prices and foreign " + "currency, the metric increased 5.8% overall and 4.8% in the U.S. Sales of food and sundries, a category " + "that includes snack foods and beverages, were up by mid single digits in the quarter, CFO Richard Galanti " + "said on the company’s earnings call. Fresh foods were up high single digits and nonfoods were up mid single " + "digits. Ancillary businesses, which includes more service-related purchases like travel, were up by low " + "single digits, he said. Costco’s food court, pharmacy and optical centers were top performers in the quarter " + "and gas was down low single digits as the price per gallon fell. More shoppers came to Costco, and they " + "spent more on their shopping trips during the quarter. Traffic increased 5.3% across the globe and 4.3% in " + "the U.S., Galanti said on the earnings call. The average ticket increased in the U.S. and worldwide, he " + "said. Inflation was roughly flat year over year in the quarter, which allowed the retailer to reduce " + "prices for some items, Galanti said. For example, he said, it’s been able to cut the price of reading " + "glasses from $18.99 to $16.99 and slash the price of a 48 count of Kirkland Signature batteries from " + "$17.99 to $15.99. In the prior quarter, he said inflation was as much as 1% year over year. Galanti said many " + "new items in categories like sporting goods and lawn and garden will also have lower prices compared with " + "a year ago because of falling freight and commodity costs. Costco has 875 warehouses, including 603 in " + "the U.S. and Puerto Rico. It also has clubs in about a dozen other countries, including Canada, Mexico, " + "Japan and China. In the second quarter, Costco opened four new clubs, including three in the U.S. " + "and one in Shenzhen, China. That marked its sixth club to open in China, Galanti said. Two of the three " + "new U.S. locations were Costco Business Centers, which are specifically geared toward small business " + "owners like restaurant operators. As of Thursday’s close, Costco shares have risen nearly 19% since the " + "start of the year. The stock touched a 52-week high of $787.08 earlier in the day and closed at $785.59, " + "bringing the company’s market value to nearly $350 billion."}, + + { + "context": "Shares of Teleperformance plunged 23% on Thursday, after the French call center and office services group " + "missed its full-year revenue target and flagged a “volatile economic environment.” Investors have been " + "spooked by the potential impact of artificial intelligence on its business model, as companies become more " + "able to tap into the technology directly for their own benefit. Teleperformance shares dropped 16% last " + "week, according to LSEG data, after Swedish financial services company Klarna said its Open AI-powered " + "customer service assistant was handling two-thirds of customer service calls. But Teleperformance CEO " + "Daniel Julien on Thursday said that AI would be a positive for its business model — and that it will never " + "fully replace the value of human interaction. “AI is part of the solutions we provide to the clients,” " + "Julien told CNBC’s “Squawk Box Europe.” “AI helps to increase the accuracy of our employees ... which is " + "great, but at the end of the day we are here to reduce the friction between the citizens, or the customer, " + "and the companies they have bought a product and service from.” He stressed, “It’s not just a transactional " + "relationship, it has a lot to do with reassuring, with trust, with empathy. So we perceive AI as enhancing " + "the job that our human employees do, but absolutely not replacing them.” hide content Teleperformance SE " + "RT Quote | Exchange | EUR 87.16 quote price arrow up+0.68 (+0.79%) Last | 03/15/24 CET WATCHLIST + QUOTE DETAILS " + "Teleperformance share price. Teleperformance reported 2.3% higher revenue at 8.345 billion euros " + "($9.091 billion) in 2023, as net profit fell year-on-year from 643 million euros to 602 million euros. " + "Diluted earnings per share hit 10.18 euros, down from 10.77 euros. In its results, the company said it is " + "working with clients on 250 AI projects, including in generative AI, and it has expanded its portfolio " + "with new partnerships in the space. “Even the most high-tech or the most AI-involved companies are clients " + "of Teleperformance. We chose that there is a complementarity and not separation,” Julien told CNBC, " + "flagging the company’s agreement with tech giant and major AI player Microsoft. “They are there to provide " + "a solution that is going to augment the productivity, augment the quality of the information that can be " + "given to the customer, but, at the end of the day, the customer is a human being. The day the customer is " + "going to be a robot, maybe AI will replace the humans.”"}, + + {"context": "CrowdStrike shares surged as much as 21% in after-hours trading Tuesday after the cybersecurity " + "company reported a beat on the top and bottom lines, plus issued stronger-than-expected guidance for " + "the upcoming quarter and full year. Here’s how the company did compared to consensus estimates " + "based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per share: 95 cents " + "adjusted vs. 82 cents expected Revenue: $845 million vs. $839 million expected For the period that " + "ended Jan. 31, CrowdStrike saw net income of $54 million, or 22 cents per share, from a " + "$48 million loss, or a 20 cent loss per share, in the year-ago period. CrowdStrike has now " + "reported GAAP net income for the past four quarters, Chief Financial Officer Burt Podbere " + "said in the earnings release. Full-year revenue rose 36% year over year, from $2.24 billion " + "to $3 billion. The company also announced it would acquire Flow Security for an undisclosed " + "price in a cash-and-stock deal, slated to close in the company’s fiscal first quarter. " + "The company has been stepping up its merger and acquisition activity in recent months. “CrowdStrike " + "is cybersecurity’s consolidator of choice, innovator of choice, and platform of choice to " + "stop breaches,” co-founder and CEO George Kurtz said in a release. The company also guided to " + "fiscal first-quarter revenue between $902 million and $906 million, better than a consensus " + "estimate of $899 million. CrowdStrike also expects earnings per share for the period between " + "89 cents and 90 cents, better than the consensus estimate of 82 cents. Podbere also reiterated " + "the company’s focus on achieving $10 billion in annual recurring revenue by 2030. The company " + "reached $3.4 billion in annual recurring revenue in January."}, + + {"context": "Shares of Amer Sports, the maker of Wilson tennis rackets and Lousiville Slugger baseball " + "bats, fell on Tuesday after the company reported strong sales in China but a slowdown in " + "wholesale orders. Here’s how the newly public athletic company did in its fourth quarter. " + "CNBC didn’t compare the results to Wall Street estimates because it’s the first earnings " + "report since Amer Sports went public. Loss per share: 25 cents Revenue: $1.32 billion In the " + "three months ended Dec. 31, the company reported a net loss of $94.9 million, or 25 cents " + "per share, compared with $148.3 million, or 39 cents per share, a year earlier. Sales rose to " + "$1.32 billion, up about 10% from $1.2 billion a year earlier. Shares closed about 5% lower. " + "Amer, which also owns Arc’teryx, Salomon, and a number of other athletic equipment and " + "apparel brands, operates in three distinct business segments. They are technical apparel, " + "which includes its pricey Arc’teryx winter jackets; outdoor performance, such as Salomon’s " + "winter sports equipment; and ball and racquet sports, which includes equipment and apparel " + "from Wilson and Louisville, among others. During the quarter, sales for Amer’s technical " + "apparel rose 26% year over year to $550 million, driven by a 42% jump in direct sales. " + "Sales in the segment primarily come from shoppers who are buying directly from Amer’s " + "brands rather than from wholesale partners. Sales for outdoor performance increased 2% to " + "$523 million, driven by strength in the segment’s winter sports equipment franchise, " + "which was offset by a slowdown in wholesale orders for Salomon footwear. Ball and racquet sales " + "declined 3% to $242 million as the segment lapped tougher comps. In the year-ago period, " + "retailers were still dealing with supply chain issues and had over-ordered equipment like " + "tennis rackets and baseball bats. As they looked to keep their inventory levels in check, " + "some wholesalers pulled back on orders during the quarter, but Amer expects the segment " + "will level out in the quarters ahead and end fiscal 2024 with sales up in the low- to " + "mid-single digit range. The company started trading on the New York Stock Exchange last " + "month under the ticker “AS.” The shares rose just 3% in Amer’s debut on the public " + "markets after it priced its IPO at a discount. Sellers showed muted interest in the " + "stock during its first day of trading over concerns about its connections and exposure to " + "China and its debt-laden balance sheet. Founded in Helsinki in 1950, Amer was a Finnish public " + "company until it was taken private in 2019 by a consortium of investors led by China’s Anta " + "Sports, FountainVest Partners, Anamered Investments and Tencent. Since the acquisition, " + "sales grew about 45% from $2.45 billion in 2020 to $3.55 billion in 2022. Revenue jumped " + "again in 2023 to $4.37 billion, the company said Tuesday."}, + + { + "context": "Shares of Dell Technologies popped more than 15% during extended trading Thursday after the company " + "released fourth-quarter results that beat analysts’ estimates and showed strong demand for its " + "artificial intelligence servers. Here’s how the company did: Earnings per share: $2.20 adjusted vs. " + "$1.73 expected by LSEG, formerly known as Refinitiv Revenue: $22.32 billion vs. $22.16 billion " + "expected by LSEG Dell’s revenue for the fiscal 2024 fourth quarter fell 11% from $25.04 billion " + "in the year-ago quarter. The company reported net income $1.16 billion, up 89% from the $614 million " + "it posted in the same period last year. Chief Financial Officer Yvonne McGill said in a release " + "that the company is increasing its annual dividend by 20% to $1.78 per share, which she called " + "a “testament to our confidence in the business.” Dell’s Infrastructure Solutions Group (ISG) " + "reported $9.3 billion in revenue for the quarter, down 6% year over year but up 10% from the " + "third quarter. Servers and networking revenue made up the bulk of that, with $4.9 billion in " + "revenue driven by “AI-optimized servers.” Storage revenue came in at $4.5 billion. The company’s " + "Client Solutions Group (CSG) reported $11.7 billion for the quarter, down 12% year over year. " + "That includes $9.6 billion in commercial client revenue, which fell 11% since the fourth quarter " + "of last year, and $2.2 billion in consumer revenue, down 19% year over year. “Our strong AI-optimized " + "server momentum continues, with orders increasing nearly 40% sequentially and backlog nearly " + "doubling, exiting our fiscal year at $2.9 billion,” Chief Operating Officer Jeff Clarke " + "said in the release. For its first quarter, Dell said during its quarterly call with " + "investors that it expects to report revenue between $21 billion and $22 billion. The company " + "said it is encouraged by momentum around AI, and that it expects to return to growth for " + "fiscal 2025. However, the company noted that the macroeconomic environment is causing some " + "customers to be cautious about infrastructure costs."}, + + {"context": "Birkenstock on Thursday beat holiday quarter revenue expectations, reporting a 22% year-on-year " + "jump, as the German sandal company benefited from higher pricing and rising U.S. demand. As a newly " + "public company, Birkenstock is still getting into a public reporting rhythm and only just " + "released its fiscal 2023 results and 2024 guidance a little over a month ago. On Thursday, " + "it said it stands by guidance issued then and still expects sales to be between 1.74 billion " + "euros and 1.76 billion euros ($1.89 billion and $1.91 billion), representing growth of 17% to 18%. " + "The shoemaker, which started trading on the New York Stock Exchange under the ticker “BIRK” in " + "October, saw a muted debut when it first hit the public markets, with shares sliding more than " + "12% on its first day as a public company. The stock has since rebounded and is up more than 5% " + "this year, as of the Wednesday close. Birkenstock’s shares closed more than 2% lower Thursday. " + "Here’s how the shoemaker did in its fiscal first quarter compared with what Wall Street was " + "anticipating, based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings per " + "share: 9 euro cents adjusted vs. 9 euro cents expected. Revenue: 302.9 million euros vs. 288.7 " + "million euros expected. The company reported a net loss of 7.15 million euros for the " + "three-month period that ended Dec. 31, or a loss of 4 euro cents per share. A year earlier, " + "it reported a loss of 9.19 million euros, or a loss of 5 euro cents per share. " + "Excluding one-time items, Birkenstock reported a profit of 17 million euros, or " + "9 euro cents per share. Sales rose to 302.9 million euros, up 22% from 248.5 million euros " + "a year earlier. Adjusted earnings before interest, taxation, depreciation and amortization " + "(EBITDA) rose 12% year on year to 81 million euros, with an adjusted EBITDA margin of 26.9%, " + "down from 29.1% a year earlier. The retailer has been making strides to grow its direct-to-consumer " + "business, which comes with better profits and more customer insights than relying on wholesale partners. " + "CEO Oliver Reichert has said the company deliberately engineers its distribution strategy so " + "demand is higher than supply but it’s working to double its production capabilities over " + "the next three years to narrow that gap. The chief executive said those investments, " + "along with other efforts the company is undertaking to drive growth, is having a “planned” " + "but “temporary” impact to profitability. The company’s gross profit margin inched down to " + "61% from 61.7% during the same period last year, with Birkenstock citing “unfavorable " + "currency translation and the planned, temporary under-absorption from our ongoing " + "capacity expansion.” The company said it continues to carefully track input costs and " + "is mitigating inflationary pressures with “executed, selective price increases.” In Europe, the " + "company said it had “two consecutive price adjustments” with “no signs of rejection.”"}, + + {"context": "Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on " + "Thursday, even as the company navigated through a period of tepid consumer electronics demand. " + "But the retailer warned of another year of softer sales and said it would lay off workers and " + "cut other costs across the business. CEO Corie Barry offered few specifics, but said the " + "company has to make sure its workforce and stores match customers’ changing shopping habits. " + "Cuts will free up capital to invest back into the business and in newer areas, such as artificial " + "intelligence, she added. “This is giving us some of that space to be able to reinvest into " + "our future and make sure we feel like we are really well positioned for the industry to " + "start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates " + "revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most " + "recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable " + "sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores " + "this year after shuttering 24 in the past fiscal year. One challenge that will affect sales " + "in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal " + "year lifted revenue by about $735 million and boosted diluted earnings per share by about " + "30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching " + "a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics " + "retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was " + "expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: " + "Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 " + "billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt " + "with slower demand in part due to the strength of its sales during the pandemic. Like " + "home improvement companies, Best Buy saw outsized spending as shoppers were stuck at " + "home. Plus, many items that the retailer sells like laptops, refrigerators and home " + "theater systems tend to be pricier and less frequent purchases. The retailer has cited other " + "challenges, too: Shoppers have been choosier about making big purchases while dealing " + "with inflation-driven higher prices of food and more. Plus, they’ve returned to " + "splitting their dollars between services and goods after pandemic years of little " + "activity. Even so, Best Buy put up a holiday quarter that was better than feared. " + "In the three-month period that ended Feb. 3, the company’s net income fell by 7% to " + "$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago " + "period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that " + "includes sales online and at stores open at least 14 months, declined 4.8% during the " + "quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater " + "setups than the year-ago period. Gaming, on the other hand, was a strong sales " + "category in the holiday quarter."} + + ] + + return earnings_releases + + +def hello_world_test(test_set, qa_model="slim-qa-gen-tiny-tool", question_type="question", temperature=0.5): + + """ Shows a basic example of generating questions and answers from a test set, and then builds a simple mini + illustrative 'model-ready' instruct database built without any manual labeling or tagging. + + -- test_set = list of dictionary entries with a single key "context" associated with the text passage + in each test entry + + -- qa_model = name of the qa gen model selected + + -- question_type = "question" | "boolean" | "multiple choice" + + -- temperature = experiment with different levels to optimize balance and variety + + """ + + # recommend using temperature of 0.2 - 0.8 - for multiple choice, use lower end of the range + + # note: if generation is very long (e.g., a summary question), then it is possible that the + # output will be malformed given the cut-off at 500 tokens - in this case, an automated remediation + # will try to resolve, but in many cases will provide an empty [] response - we have also observed + # some repetition in very long generations too + + qa_model = ModelCatalog().load_model(qa_model, sample=True, temperature=temperature, max_output=500, + get_logits=False) + + qa_pair_set = [] + ds = [] + + print(f"\nRun Q-A Gen Inferences on Test Set") + + for text_passage in test_set: + + response = qa_model.function_call(text_passage["context"], params=[question_type], get_logits=False) + + # expect response in the form of: + # -- "llm_response": {'question': ['generated question?'], 'answer': ['answer to question']} + + if response: + if "llm_response" in response: + if "question" in response["llm_response"] and "answer" in response["llm_response"]: + + # get the question and answer from the llm response + new_q = response["llm_response"]["question"] + new_a = response["llm_response"]["answer"] + + # keep only where there is both a non-empty question and answer + if new_q and new_a: + qa_pair_set.append({"question": new_q[0], "answer": new_a[0]}) + ds.append(({"question": new_q[0], "answer": new_a[0], "context": text_passage["context"]})) + + print(f"inference - response: ", response) + + print("\nShow list of question-answer pairs created") + + for i, qa_pair in enumerate(qa_pair_set): + + print(f"new generated question-answer pairs: {i} - {qa_pair}") + + print("\nBuild model-ready 'mini' instruct dataset") + + # use phi-3 instruct wrapper template as separators between elements + # --easy to substitute these separators for other popular templates + sep1= "<|user|>\n" + sep2 = "<|end|>\n" + sep3 = "<|assistant|>\n" + + model_ready_ds = [] + for i, sample in enumerate(ds): + new_sample = sep1 + sample["context"] + "\n" + sample["question"] + sep2 + sep3 + sample["answer"] + model_ready_ds.append({"text": new_sample, "source": "earnings_test_set_example"}) + + random.shuffle(model_ready_ds) + fp = os.path.join(LLMWareConfig().get_llmware_path(), "instruct_ds_example.jsonl") + + train_file = open(fp, "w") + for rows in model_ready_ds: + jsonl_row = json.dumps(rows) + train_file.write(jsonl_row) + train_file.write("\n") + + train_file.close() + + print(f"\ncreated dataset @: {fp}") + + return qa_pair_set + + +if __name__ == "__main__": + + # get the earnings release test set (list of dicts with "context" key) + test_set = earning_releases_test_set() + + # set a max generation on the GGUF engine at 1000 tokens - each model can be loaded up to this amount + GGUFConfigs().set_config("max_output_tokens", 1000) + + # run the main example + hello_world_test(test_set,qa_model="slim-qa-gen-tiny-tool",question_type="question", temperature=0.5) + diff --git a/solutions/slim_agents/using-slim-sa-ner-combo.py b/solutions/slim_agents/using-slim-sa-ner-combo.py new file mode 100644 index 0000000..ad6bc6f --- /dev/null +++ b/solutions/slim_agents/using-slim-sa-ner-combo.py @@ -0,0 +1,17 @@ + +""" This example illustrates how to use the slim-sa-ner-tool, which is a 'combination' model that brings together + both sentiment and named entity recognition in a single function-calling model. """ + +from llmware.models import ModelCatalog + + +model = ModelCatalog().load_model("slim-sa-ner-tool", sample=False, temperature=0.0) + +text = ("Tesla stock declined yesterday 8% in premarket trading after a poorly-received event in San Francisco " + "yesterday, in which the company indicated a likely shortfall in revenue.") + +response = model.function_call(text, function="classify", params=["sentiment,person,organization,place"]) + +print("response: ", response) + +ModelCatalog().tool_test_run("slim-sa-ner-tool") diff --git a/solutions/slim_agents/using_slim_boolean_model.py b/solutions/slim_agents/using_slim_boolean_model.py new file mode 100644 index 0000000..18b3a6a --- /dev/null +++ b/solutions/slim_agents/using_slim_boolean_model.py @@ -0,0 +1,31 @@ + +""" This example illustrates how to use the slim-boolean model. In this case, we will use test documents provided + in the model package, which will be cached locally the first time that you load the model. """ + +from llmware.models import ModelCatalog + +# load the model - best results with sample=False, but feel free to experiment +model = ModelCatalog().load_model("slim-boolean-tool",sample=False, temperature=0.0, max_output=200) + +# get the test set packaged with the model +test_set = ModelCatalog().get_test_script("slim-boolean-tool") + +# iterate through the test set +for i, sample in enumerate(test_set): + + # add optional "explain" parameter to each question + question = sample["question"] + " (explain)" + + # key line: invoke function call on the model, with boolean function, and pass the question as the parameter + response = model.function_call(sample["context"], function="boolean", params=[question]) + + print("response: ", response) + + # analyze the logits + analysis = ModelCatalog().get_fx_scores(response,"slim-boolean-tool") + + # display to the screen + print("\nllm_response: ", i, question, response["llm_response"]) + print("analysis: ", analysis) + + diff --git a/solutions/slim_agents/using_slim_summary.py b/solutions/slim_agents/using_slim_summary.py new file mode 100644 index 0000000..a8768d8 --- /dev/null +++ b/solutions/slim_agents/using_slim_summary.py @@ -0,0 +1,45 @@ + +""" This example illustrates how to use slim-summary models to easily create summaries generated as a list of +summary points for easy integration into multi-step workflows. There are several slim-summary function-calling +models available in different sizes and based on leading underlying base models. """ + +from llmware.models import ModelCatalog + +# three slim-summary function calling models available + +slim_summary_models = ["slim-summary-tool", # original - stablelm-3b + "slim-summary-tiny-tool", # small - tiny-llama (1.1b) + "slim-summary-phi-3-gguf" # phi-3 - phi-3 (3.8b) + ] + +# load the model and set the sampling and output parameters +model = ModelCatalog().load_model("slim-summary-tool", + sample=False, + temperature=0.0, + max_output=200) + + +# get the test data set packaged with the model +test_script = ModelCatalog().get_test_script("slim-summary-tool") + +# iterate through the samples +for i, sample in enumerate(test_script): + + # invoke function call on the model, passing the context passage and the 'summarize' function + # the parameter can be a generic phrase, e.g., 'key points' or 'brief description' or 'summary' + # if the material has a lot of numeric data points, try the parameter 'data points' or 'financial data points' + # if you are looking for a single line of output, try 'brief description' + # the number in ( ) is optional - but is intended to guide the model to provide with a list with the requested + # number of elements + + response = model.function_call(sample["context"], function="summarize", params=["data points (5)"]) + + # display the response + print("\nresponse: ", response) + + # check how effectively the model mapped the output to the requested number of points + r = response["llm_response"] + for j, entries in enumerate(r): + print("summary points: ", j, entries) + + diff --git a/solutions/slim_agents/using_slim_xsum.py b/solutions/slim_agents/using_slim_xsum.py new file mode 100644 index 0000000..7defa91 --- /dev/null +++ b/solutions/slim_agents/using_slim_xsum.py @@ -0,0 +1,24 @@ + +""" This example illustrates how to use the slim-xsum model to provide 'extreme summarizations', generally + providing a headline or title (typically no more than 10-15 words) to describe a longer text passage. """ + +from llmware.models import ModelCatalog + +slim_xsum_models = ["slim-xsum-tool", "slim-xsum-phi-3-gguf"] + +# load the model +model = ModelCatalog().load_model("slim-xsum-phi-3-gguf",sample=False, temperature=0.0, max_output=200) + +# load the test dataset packaged with the model (will be cached locally the first time you load) +test_set = ModelCatalog().get_test_script("slim-xsum-phi-3-gguf") + +# iterate through the test set samples +for i, sample in enumerate(test_set): + + # invoke function call on the model, and pass 'xsum' as the parameter + response = model.function_call(sample["context"], params=["xsum"]) + + # display the output + print("\nllm_response: ", i, response["llm_response"]) + + diff --git a/solutions/slim_agents/youtube_topics_extraction.py b/solutions/slim_agents/youtube_topics_extraction.py new file mode 100644 index 0000000..84ff7f4 --- /dev/null +++ b/solutions/slim_agents/youtube_topics_extraction.py @@ -0,0 +1,71 @@ +""" +This script demonstrates how to extract topics from a youtube video transcript using the LLMfx agent and the slim topics tool. +Transcripts are obtained by video ID through the youtube_transcript_api, saved to a text file, and then parsed by the LLMfx agent. +""" + +from llmware.parsers import Parser +from llmware.agents import LLMfx +from llmware.setup import Setup +import os +from youtube_transcript_api import YouTubeTranscriptApi +from youtube_transcript_api.formatters import TextFormatter + + + +def transcript_parser(file_dest, file_name): + # Given the filename and filepath, parses pdf into chunks + + doc_chunks = Parser().parse_one_text(file_dest, file_name) + print("number of chunks: ", len(doc_chunks)) + + # Create a LLMfx object + agent = LLMfx() + # Load in the chunked document. To make the demo run faster or to test, slice it with [0:5] + agent.load_work(doc_chunks) + # Load in the topic tool + agent.load_tool_list(["topics"]) + + funcall_list = [] + while True: + funcall_list.append(agent.exec_multitool_function_call(["topics"])) + + if not agent.increment_work_iteration(): + break + + return funcall_list + +# Function to collapse the report to show unique topics +def collapser(report): + no_duplicates = [] + # Raise this to make your program more selective, lower it to get more topics + required_confidence_score = 0.1 + for entry in report: + if entry[0]['confidence_score'] > required_confidence_score: + if 'topics' not in entry[0]['llm_response']: + continue + for topics in entry[0]['llm_response']['topics']: + if topics not in no_duplicates: + no_duplicates.append(topics) + return no_duplicates + +if __name__ == "__main__": + #Edit the Video ID to the desired video + video_id = "a62ghRjnLFU" + formatter = TextFormatter() + transcript = YouTubeTranscriptApi.get_transcript(video_id) + text_formatted = formatter.format_transcript(transcript) + +# Save the formatted text to a file + file_name = "transcript.txt" + file_dest = os.getcwd() + file_path = os.path.join(file_dest, file_name) + print("Transcript saved to: ", file_path) + with open(file_path, "w") as file: + file.write(text_formatted) + + + analysis = transcript_parser(file_dest, file_name) + print("\n Analysis: Shows Topics located in each chunk of the Document \n") + print(analysis) + print("\n Collapsed Analysis: Shows Unique topics located over the entire Document that meet the required confidence score. \n") + print(collapser(analysis)) diff --git a/solutions/sources/README.md b/solutions/sources/README.md new file mode 100644 index 0000000..3a0704e --- /dev/null +++ b/solutions/sources/README.md @@ -0,0 +1,56 @@ + 🚀 Parsing Examples 🚀 +=============== + +**Parsing is the Humble Hero of Good RAG Pipelines** + +LLMWare supports parsing of a wide range of unstructured content types, and views parsing, text chunking and indexing as the first step in the pipeline, and like any pipeline, care and attention to getting "great input" is usually the key to "great output." + +In this repository, we show several key features of parsing with llmware: + + +**Parsing PDFs like a Pro** + +- Configuring text chunking and extraction parameters - [**PDF Configuration**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/pdf_parser_new_configs.py) + +- PDF Table extraction - [**PDF Table**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/pdf_table_extraction.py) + +- Fallback to OCR - [**PDF-by-OCR**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_pdf_by_ocr.py) + + +**Parsing Office Documents (Powerpoints, Word, Excel)** + +- Configuring text chunking and extraction parameters - [**Office Configuration**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/office_parser_new_configs.py) + +- Handling ZIPs and mixed file types - [**Microsoft IR Documents**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parsing_microsoft_ir_docs.py) + +- Running OCR on Images Extracted - [**OCR Embedded Doc Images**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/ocr_embedded_doc_images.py) + + +**Parsing without a Database** + +- Parse in Memory - [**Parse in Memory**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_in_memory.py) + +- Parse directly into a Prompt - [**Parse in Prompt**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_into_prompt.py) + +- Parse to JSON file - [**Parse to JSON**](https://github.com/llmware-ai/llmware/blob/6936ef404487a0fc88e2571648c7181bd7c07ad5/examples/Parsing/parse_to_json.py) + + +**Other Content Types** + +- Custom CSV - [**Custom CSV files**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_csv_custom.py) + +- Custom JSON - [**Custom JSON files**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_jsonl_custom.py) + +- Images - [**OCR on Images**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_images.py) + +- Web/HTML - [**Website Extraction**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Parsing/parse_web_sources_in_memory.py) + +- Voice (WAV) - in Use_Cases - [**Parsing Great Speeches**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/parsing_great_speeches.py) + + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + +### **Let's get started! 🚀** + + diff --git a/solutions/sources/author_filter.py b/solutions/sources/author_filter.py new file mode 100644 index 0000000..46eb161 --- /dev/null +++ b/solutions/sources/author_filter.py @@ -0,0 +1,54 @@ + +"""This example demonstrates the various ways to retrieve data from libraries: + 1. Create a sample library (e.g., UN Resolutions) + 2. Execute a Text Query with Author Name Filter + 3. Display Results +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def create_un_500_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "UN-Resolutions-500") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +def text_retrieval_by_author(library, query_text, author): + + # create a Query instance and pass the previously created Library object + query = Query(library) + + # set the keys that should be returned in the results + query.query_result_return_keys = ["file_source", "page_num", "text", "author_or_speaker"] + + # perform a text query by author + query_results = query.text_query_by_author_or_speaker(query_text, author) + + # display the results + for i, result in enumerate(query_results): + + file_source = result["file_source"] + page_num = result["page_num"] + author = result["author_or_speaker"] + text = result["text"] + + # shortening for display purpose only + if len(text) > 150: text = text[0:150] + " ... " + + print (f"\n> Top result for '{query_text}': {file_source} (page {page_num}), Author: {author}:\nText:{text}") + + return query_results + + +if __name__ == "__main__": + + library = create_un_500_sample_library("lib_author_filter_1") + qr_output = text_retrieval_by_author(library=library, query_text='United Nations', author='Andrea Chambers') diff --git a/solutions/sources/bibliography.py b/solutions/sources/bibliography.py new file mode 100644 index 0000000..b3212a2 --- /dev/null +++ b/solutions/sources/bibliography.py @@ -0,0 +1,49 @@ + +"""This example demonstrates the various ways to retrieve data from libraries: + 1. Create a sample 'FinDocs' library + 2. Execute a search (text in this case - but could be any query) + 3. Generate bibliography of all of the query results +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def create_fin_docs_sample_library(library_name): + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "FinDocs") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +# A bibliography is the list of documents and their pages referenced in a set of query results. +# The format is: [{'Gaia EXECUTIVE EMPLOYMENT AGREEMENT.pdf': [3, 5, 2, 4, 1]}] + +def get_bibliography_from_query_results(library, query_text): + + # Create a Query instance + query = Query(library) + + # Perform a simple query + query_results = query.text_query(query_text, result_count=20, exact_mode=False) + + # Get a bibliography + biblio = query.bibliography_builder_from_qr(query_results=query_results) + + # Print out the bibliography + print (f"\n> Bibliography for '{query_text}':\n{biblio}") + + return biblio + + +if __name__ == "__main__": + + library = create_fin_docs_sample_library("lib_biblio_1") + query_text = "amazon web services" + biblio = get_bibliography_from_query_results(library,query_text) + + print("update: biblio created - ", biblio) diff --git a/solutions/sources/bulk_retrieval.py b/solutions/sources/bulk_retrieval.py new file mode 100644 index 0000000..4eb98bb --- /dev/null +++ b/solutions/sources/bulk_retrieval.py @@ -0,0 +1,46 @@ + +"""This example demonstrates the various ways to retrieve data from libraries: + 1. Basic retrieval + 2. Retrieval with filters + 3. Bulk retrieval + 4. Retrieval State and Export +""" + +import os +from llmware.retrieval import Query +from llmware.library import Library +from llmware.setup import Setup + + +def create_agreements_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "Agreements") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +# Sometimes you want to retrieve all data so you can further process it yourself +def perform_bulk_retrieval(library): + + # Create a Query instance + query = Query(library) + + # Create a list of keys of interest. This can be omitted if you want all keys + key_dict = ["file_source", "text", "page_num", "author_or_speaker"] + + # Get the whole libary. This returns a list of all blocks + all_blocks = query.get_whole_library(selected_keys=key_dict) + + print (f"\n> Bulk retrieval Retrieval'") + print (f"\n{len(all_blocks)} blocks were retrieved") + + return all_blocks + + +if __name__ == "__main__": + + library = create_agreements_sample_library("lib_agreements_1") + perform_bulk_retrieval(library) diff --git a/solutions/sources/create_custom_table-1.py b/solutions/sources/create_custom_table-1.py new file mode 100644 index 0000000..6fa38eb --- /dev/null +++ b/solutions/sources/create_custom_table-1.py @@ -0,0 +1,120 @@ + +""" This example shows the basic recipe for creating a CustomTable with LLMWare and a few of the basic methods + to quickly get started. + + In this example, we will build a very simple 'hello world' Files table, which we will build upon in a future + example by aggregating a more interesting and useful set of attributes from a LLMWare Library collection. + + CustomTable is designed to work with the text collection databases supported by LLMWare: + + SQL DBs --- Postgres and SQLIte + NoSQL DB --- Mongo DB + + Even though Mongo does not require a schema for inserting and retrieving information, the CustomTable method + will expect a defined schema to be provided (good best practice, in any case). """ + +from llmware.resources import CustomTable + + +def hello_world_custom_table(): + + # simple schema for a table to track Files/Documents + # note: the schema is a python dictionary, with named keys, and the value corresponding to the data type + # for sqlite and postgres, any standard sql data type should generally work + + files_schema = {"custom_doc_num": "integer", + "file_name": "text", + "comments": "text"} + + # create a CustomTable object + db_name = "sqlite" + table_name = "files_table_1000" + ct = CustomTable(db=db_name,table_name=table_name, schema=files_schema) + + # insert a few sample rows - each row is a dictionary with keys from the schema, and the *actual* values + r1 = {"custom_doc_num": 1, "file_name": "technical_manual.pdf", "comments": "very useful overview"} + ct.write_new_record(r1) + + r2 = {"custom_doc_num": 2, "file_name": "work_presentation.pptx", "comments": "need to save for future reference"} + ct.write_new_record(r2) + + r3 = {"custom_doc_num": 3, "file_name": "dataset.json", "comments": "will use in next project"} + ct.write_new_record(r3) + + # to see the entries - pull all items from the table + all_results = ct.get_all() + + print("\nTEST #1 - Retrieving All Elements") + for i, res in enumerate(all_results): + print("results: ", i, res) + + # look at the database schema + schema = ct.get_schema() + + print("\nTEST #2 - Getting the Table Schema") + print("schema: ", schema) + + schema_str = ct.sql_table_create_string() + + print("table create sql: ", schema_str) + + # perform a basic lookup with 'key' and 'value' + f = ct.lookup("custom_doc_num", 2) + + print("\nTEST #3 - Basic Lookup - 'custom_doc_num' = 2") + print("lookup: ", f) + + # if you prefer SQL, pass a SQL query directly (note: this will only work on Postgres and SQLite) + + if db_name == "sqlite": + + # note: our standard 'unpacking' of a row of sqlite includes the rowid attribute + custom_query = f"SELECT rowid, * FROM {table_name} WHERE custom_doc_num = 3;" + + elif db_name == "postgres": + custom_query = f"SELECT * FROM {table_name} WHERE custom_doc_num = 3;" + + elif db_name == "mongo": + custom_query = {"custom_doc_num": 3} + else: + print("must use either sqlite, postgres or mongo") + return -1 + + cf = ct.custom_lookup(custom_query) + + print("\nTEST #4 - Custom SQL Lookup - 'custom_doc_num' = 3") + print("custom query lookup: ", cf) + + print("\nTEST #5 - Making Updates and Deletes") + + # to delete a record + ct.delete_record("custom_doc_num", 1) + print("deleted record") + + # to update the values of a record + ct.update_record({"custom_doc_num": 2}, "file_name", "work_presentation_update_v2.pptx") + print("updated record") + + updated_all_results = ct.get_all() + + for i, res in enumerate(updated_all_results): + print("updated results: ", i, res) + + print("\nTEST #6 - Delete Table - uncomment and set confirm=True") + # done? delete the table and start over + # -- note: confirm=True must be set + # ct.delete_table(confirm=False) + + # look at all tables in the database + tables = ct.list_all_tables() + + print("\nTEST #7 - View all of the tables on the DB") + for i, t in enumerate(tables): + print("tables:" ,i, t) + + return 0 + + +if __name__ == "__main__": + + hello_world_custom_table() diff --git a/solutions/sources/customer_table.csv b/solutions/sources/customer_table.csv new file mode 100644 index 0000000..2a54be1 --- /dev/null +++ b/solutions/sources/customer_table.csv @@ -0,0 +1,20 @@ +customer_name,account_number,customer_level,vip_customer,annual_spend,user_name +Martha Williams,98320893,gold,yes,63250,mwilliams +Susan Soinsin,53439382,silver,no,112 ,ssinsin +Michael Rogers,88888444,bronze,no,3 ,rogersm +John Jones,9898482,gold,yes,12430,jjns +Melinda Lyons,1234953,silver,no,234,mlyons +Arvind Arora,8998899,gold,yes,42650,aarora +Paul Rinno,3829235,silver,no,1293,prinno +Vinod Aggarwal,9389532,platinum,yes,93540,aggarwal +Bryan Lewis,12399853,gold,no,9732,brlewis +Allison Winters,83902867,gold,yes,15000,wintersall +Wyatt Rivers,73849532,bronze,no,18,wyariv +Olivia Smith,8372639,gold,yes,22500,osmith +Alan Wang,5478320,silver,yes,6000,wangalan +Olivier Dubois,80839204,bronze,no,27,odubois +Wilmer Rodriguez,3674895,platinum,yes,76382,rodwilm +Eliza Listron,8763092,gold,yes,14593,listrone +Kim Yi,3278073,silver,no,873,yikim +Tom Marshall,78368037,silver,no,758 ,marshallt +Douglas Hudson,2378490,gold,yes,10000,huddoug \ No newline at end of file diff --git a/solutions/sources/document_filters.py b/solutions/sources/document_filters.py new file mode 100644 index 0000000..487c8f8 --- /dev/null +++ b/solutions/sources/document_filters.py @@ -0,0 +1,73 @@ + +""" +This 'getting started' example demonstrates how to use basic text retrieval with the Query class + 1. Create a sample library + 2. Run a basic text query + 3. View the results +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + +# A very powerful form of retrieval involves document filters. Once a 'document filter' is created, it can be +# applied to query further only in that document set +# For example:You could set up a document filter to get all documents that mention +# a topic like 'Artificial Intelligence' +# and then within that subset of documents, look for details on leading researchers. + + +def create_agreements_embeddings_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "Agreements") + parsing_output = library.add_files(ingestion_folder_path) + + library.install_new_embedding(embedding_model_name="mini-lm-sbert", vector_db="chromadb") + + return library + + +def perform_retrieval_with_document_filters(library, doc_filter_text, query_text): + + print(f"Example - using a Document Filter with Semantic Search") + + # create a Query instance + query = Query(library) + + # create a document filter using exact (text) search mode + doc_filter = query.document_filter(doc_filter_text, query_mode="text", exact_mode=True) + + # the document filter narrows the library to only documents with the target filter_text + print("\ndocument filter: ", doc_filter) + + # perform a semantic query with the document filter -> results will be limited to the documents in the filter + semantic_results = query.semantic_query_with_document_filter(query_text, doc_filter) + + # display the text from the results + print (f"\nRetrieval with a document filter'") + for i, result in enumerate(semantic_results): + result_text = result["text"] + print (f"Results - {i} - file_source - {result['file_source']} - {result['distance']}" + f" - {result['text']}") + + return 0 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + lib = create_agreements_embeddings_sample_library("lib_doc_filter_1") + + # this will run a two-step query + # step 1 - look in the entire library for documents with the executive's name 'leto apollo' + # step 2 - using that document filter, pass to a semantic query, and retrieve specific text chunks + # from the selected documents with 'base salary' + + perform_retrieval_with_document_filters(library=lib, doc_filter_text="leto apollo", query_text='base salary') + + diff --git a/solutions/sources/dual_pass_with_custom_filter.py b/solutions/sources/dual_pass_with_custom_filter.py new file mode 100644 index 0000000..a7d7853 --- /dev/null +++ b/solutions/sources/dual_pass_with_custom_filter.py @@ -0,0 +1,92 @@ +import os +from llmware.library import Library +from llmware.setup import Setup +from llmware.status import Status +from llmware.retrieval import Query + +def create_unr500_sample_library(library_name): + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "UN-Resolutions-500") + parsing_output = library.add_files(ingestion_folder_path) + return library + +def parse_and_generate_vector_embeddings(library_name, input_folder_path): + # Configuration for embeddings + embedding_model = "mini-lm-sbert" + vector_db = "milvus" + + # Create a new library or load if it exists + print(f"\nCreating or loading library: {library_name}") + library = Library().load_library(library_name) + + if not library: + print(f"Error: Failed to create or load library '{library_name}'") + return + + # Parse and text index files + print("Parsing and Text Indexing Files") + if not os.path.exists(input_folder_path): + print(f"Error: Input folder path '{input_folder_path}' does not exist.") + return + + num_files_added = library.add_files(input_folder_path=input_folder_path) + print(f"Number of files added for indexing: {num_files_added}") + + # Generate embeddings + print(f"Generating Embeddings in {vector_db} db with Model {embedding_model}") + embedding_status = library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db) + if not embedding_status: + print(f"Error: Embedding generation failed for model '{embedding_model}' with vector db '{vector_db}'") + return + + # Check the embedding status + update = Status().get_embedding_status(library_name, embedding_model) + print("Embeddings Complete - Status check at end of embedding: ", update) + + +def perform_dual_pass_query(library, query_text, result_count=20, primary="text", custom_filter=None, desired_match_status=None): + print(f"\nPerforming dual pass query: '{query_text}'") + + q = Query(library) + + # Perform the query with the custom filter + dual_pass_results = q.dual_pass_query(query_text, result_count=result_count, primary=primary, custom_filter=custom_filter) + + # Filter results based on match status if desired_match_status is specified + if desired_match_status: + dual_pass_results = [result for result in dual_pass_results if result.get('match_status') == desired_match_status] + + for i, result in enumerate(dual_pass_results): + print(f"Result {i+1}:") + print(f" Doc ID: {result.get('doc_ID')}") + print(f" Block ID: {result.get('block_ID')}") + print(f" Page Num: {result.get('page_num')}") + print(f" Text Snippet: {result.get('text')}") + print(f" File Source: {result.get('file_source')}") + print(f" Match Status: {result.get('match_status')}") + print("----") + + return dual_pass_results + +if __name__ == "__main__": + + library_name = "un_resolutions500" + query_text = "what weapons will cause an arms race?" # Replace with your query text + + # Custom filter definition + custom_filter = { + "author_or_speaker": "ETPU FrontDesk1", + "file_type": "pdf", + "content_type": "image" + # Add more filters as needed + } + + # Assuming library is already created and vector embeddings are generated + library = Library().load_library(library_name) + + # Perform a dual pass query on the library with custom filter + dual_pass_results = perform_dual_pass_query(library, query_text, custom_filter=custom_filter) + + # If you want to filter by match_status, pass the desired status like "matched" in the function call + # dual_pass_results = perform_dual_pass_query(library, query_text, custom_filter=custom_filter, desired_match_status="matched") diff --git a/solutions/sources/loading_csv_into_custom_table-2a.py b/solutions/sources/loading_csv_into_custom_table-2a.py new file mode 100644 index 0000000..4201fc9 --- /dev/null +++ b/solutions/sources/loading_csv_into_custom_table-2a.py @@ -0,0 +1,72 @@ + +""" This example shows how to quickly build a CustomTable using a 'pseudo-DB' CSV file. A 'pseudo-DB' is a CSV + organized in a set of rows with a common column structure. Below we will show a few tools to analyze and + validate the CSV upfront to assess if there are areas that need remediation before attempting to safely load + into a database. + + CustomTable is designed to work with the text collection databases supported by LLMWare: + + SQL DBs --- Postgres and SQLIte + NoSQL DB --- Mongo DB + + Even though Mongo does not require a schema for inserting and retrieving information, the CustomTable method + will expect a defined schema to be provided (good best practice, in any case). """ + +from llmware.resources import CustomTable + + +def building_custom_table_from_csv(): + + # point fp and fn at the file_path of the CSV file + fp = "/path/to/your/csv_file" + + # good example in examples folder - customer_table.csv + fn = "sample_file.csv" + + # first analyze the csv and confirm that the rows and columns are consistently being extracted + analysis = CustomTable().validate_csv(fp,fn,delimiter=',',encoding='utf-8-sig') + + print("\nAnalysis of the CSV file") + for key, value in analysis.items(): + print(f"analysis: {key} - {value}") + + table_name = "sample_table_100" + + # use "postgres" | "mongo" | "sqlite" + db_name = "postgres" + + ct = CustomTable(db=db_name,table_name=table_name) + + # load the csv, which will identify the schema and data types, and package as 'rows' ready for db insertion + # -- this method will NOT create the DB table or insert any rows - that happens in the next step + # -- if there is a 'header_row', then it will not be inserted in the DB (so row count may differ by 1 + + output = ct.load_csv(fp,fn) + + print("\nLoad CSV output") + for key, value in output.items(): + print(f"output: {key} - {value}") + + # spot-check the rows that have been created before inserting into database as a final check + print("\nSpot-Check Rows Before Inserting into DB Table") + sample_size = min(len(ct.rows), 10) + for x in range(0,sample_size): + print("rows: ", x, ct.rows[x]) + + # when ready, uncomment, and insert the rows into the DB + ct.insert_rows() + + # basic query + # e.g., if using customer_table included in example folder - "customer_name", "Martha Williams" + customer = ct.lookup("key", "lookup_value") + print("\nLookup from DB") + print(f"customer_record: ", customer) + + ct.delete_table(confirm=True) + + return 0 + + +if __name__ == "__main__": + + building_custom_table_from_csv() diff --git a/solutions/sources/loading_csv_w_config_options-2b.py b/solutions/sources/loading_csv_w_config_options-2b.py new file mode 100644 index 0000000..619eec4 --- /dev/null +++ b/solutions/sources/loading_csv_w_config_options-2b.py @@ -0,0 +1,129 @@ + +""" This example illustrates how to use the various configuration options to maximize the quality of CSV files + loading into a custom DB table. For basic getting started examples, please see "create_custom_table.py" and + "loading_csv_into_custom_table.py" in this examples repository first. + + CustomTable is designed to work with the text collection databases supported by LLMWare: + + SQL DBs --- Postgres and SQLIte + NoSQL DB --- Mongo DB + + Even though Mongo does not require a schema for inserting and retrieving information, the CustomTable method + will expect a defined schema to be provided (good best practice, in any case). """ + +from llmware.resources import CustomTable + + +def building_custom_table_from_csv(config_option=2): + + # point fp and fn at the file_path of the CSV file + fp = "/path/csv/file" + + # note: this example uses the "customer_table.csv" example file found in the examples repository + # -- if substituting for your own csv, please also adjust the sample query below + fn = "customer_table.csv" + + # first analyze the csv and confirm that the rows and columns are consistently being extracted + analysis = CustomTable().validate_csv(fp ,fn ,delimiter=',' ,encoding='utf-8-sig') + + print("\nAnalysis of the CSV file") + for key, value in analysis.items(): + print(f"analysis: {key} - {value}") + + if not (1 <= config_option <= 5): + print("\nsetting config to default == 1") + config_option = 1 + + table_name = "customer_table_1000" + db_name = "sqlite" + output = None + + # loading a csv into a database has three main steps + # 1. construct CustomTable object + # 2. load_csv - *** where most of the configuration will occur *** + # 3. insert_rows + + ct = CustomTable(db=db_name,table_name=table_name) + + if config_option == 1: + + # load_csv - Option #1 - this is the simplest case + # -- will view the first row as a "header row" and use to derive the column names for the schema + # -- will use the first row after the header_row as a 'test row' and apply a simple mechanical test to + # infer the data type for each column as either 'text' | 'integer' | 'float' + + output = ct.load_csv(fp ,fn) + + elif config_option == 2: + + # load_csv - Option #2 - pass a set of column names and use as the basis for the schema + # -- assumption is that the list of column names will be the same length as the # of columns + # -- if there is a header row, it will be skipped. + # -- If no header_row and data starts at row 0, then set header_row = False + # -- will use the first row after the header row to try to infer the data type automatically + + # note: if using the query example below, change "customer_name" key to "CUST_NAME" + + cols = ["CUST_NAME", "ACCT_NUM", "LEVEL", "VIP","SPEND","UNAME"] + output = ct.load_csv(fp, fn, column_names=cols, header_row=True) + + elif config_option == 3: + + # load_csv - Option #3 - pass a specific mapping of column names and column indices + # in this case, the schema will be passed on the col names in the col_mapping - and can be a + # subset of the total number of columns + # the ordinal indices are 0th-indexed, and correspond to the column numbers that should be pulled + + # note: if using the query example below, change "customer_name" key to "CUST_NAME" + + col_mapping = {"CUST_NAME": 0, "customer_number" :1, "spend": 4} + output = ct.load_csv(fp, fn, column_mapping_dict=col_mapping) + + elif config_option == 4: + + # load_csv - Option #4 - pass an explicit data type mapping, for all or some columns, + # which will 'over-ride' the estimation + + dt_mapping = {1: "decimal", 4: "text"} + output = ct.load_csv(fp, fn, data_type_map=dt_mapping) + + elif config_option == 5: + + # load_csv - Option #5 - adjust encoding and delimiter + + # note: this option is not intended for use with the customer_table.csv example, but can be used for + # csv that are tab separated, or have different encoding expectations + + output = ct.load_csv(fp, fn, encoding="latin-1", delimiter="\t") + + print("\nLoad CSV output") + for key, value in output.items(): + print(f"output: {key} - {value}") + + # spot-check the rows that have been created before inserting into database as a final check + print("\nSpot-Check Rows Before Inserting into DB Table") + sample_size = min(len(ct.rows), 10) + for x in range(0 ,sample_size): + print("rows: ", x, ct.rows[x]) + + # when ready, uncomment, and insert the rows into the DB + ct.insert_rows() + + # basic query + if config_option == 2 or config_option == 3: + customer_key = "CUST_NAME" + else: + customer_key = "customer_name" + + customer = ct.lookup(customer_key, "Martha Williams") + print("\nLookup from DB") + print(f"customer_record: ", customer) + + return 0 + + +if __name__ == "__main__": + + # to see how different config options operate, then change the config_option between 1-5 + + building_custom_table_from_csv(config_option=2) diff --git a/solutions/sources/loading_json_custom_table-3a.py b/solutions/sources/loading_json_custom_table-3a.py new file mode 100644 index 0000000..0471370 --- /dev/null +++ b/solutions/sources/loading_json_custom_table-3a.py @@ -0,0 +1,67 @@ + +""" This example shows how to quickly build a CustomTable using a 'pseudo-DB' JSON/JSONL file. A 'pseudo-DB' is a + well-formed JSON/JSONL file that has a common set of keys in each dictionary entry, and multiple repeating + 'row-like' entries that can be iterated through and converted into a row/column database structure. Below we + will show a few tools to analyze and validate the JSON/JSONL upfront to assess if there are areas that + need remediation before attempting to safely loading into a database. + + CustomTable is designed to work with the text collection databases supported by LLMWare: + + SQL DBs --- Postgres and SQLIte + NoSQL DB --- Mongo DB + + Even though Mongo does not require a schema for inserting and retrieving information, the CustomTable method + will expect a defined schema to be provided (good best practice, in any case). """ + +from llmware.resources import CustomTable + + +def building_custom_table_from_json(): + + # point fp and fn at the file_path of the JSON/JSONL file + fp = "/local_path/to/json_files" + + # good example file in examples folder path - "model_list.json" + fn = "my_test_file.json" + + # first analyze the json and confirm that the rows and columns are consistently being extracted + analysis = CustomTable().validate_json(fp,fn,key_list=None) + + print(f"\nAnalysis of JSON/JSONL file") + for key, value in analysis.items(): + print(f"analysis: {key} - {value}") + + table_name = "example_json_table_100" + + # use any of "mongo" | "sqlite" | "postgres" + db_name = "mongo" + + ct = CustomTable(db=db_name,table_name=table_name) + + output = ct.load_json(fp,fn) + + print(f"\nOutput from load_json") + for key, value in output.items(): + print(f"load_json: {key} - {value}") + + # spot-check the rows that have been created before inserting into database as a final check + print("\nSpot-Check Rows Before Inserting into DB Table") + sample_size = min(len(ct.rows), 10) + for x in range(0,sample_size): + print("rows: ", x, ct.rows[x]) + + # when ready, uncomment, and insert the rows into the DB + ct.insert_rows() + + # lookup - if using the model_list.json sample file, use "model_name", "slim-extract-tool" + res = ct.lookup("key", "selected_value") + + print("\nLookup Test") + print("result: ", res) + + return 0 + + +if __name__ == "__main__": + + building_custom_table_from_json() diff --git a/solutions/sources/loading_json_w_config_options-3b.py b/solutions/sources/loading_json_w_config_options-3b.py new file mode 100644 index 0000000..5fbf18f --- /dev/null +++ b/solutions/sources/loading_json_w_config_options-3b.py @@ -0,0 +1,106 @@ + +""" This example illustrates how to use the various configuration options to maximize the quality of JSON/JSONL files + loading into a custom DB table. For basic getting started examples, please see "create_custom_table.py" and + "loading_json_into_custom_table.py" in this examples repository first. + + CustomTable is designed to work with the text collection databases supported by LLMWare: + + SQL DBs --- Postgres and SQLIte + NoSQL DB --- Mongo DB + + Even though Mongo does not require a schema for inserting and retrieving information, the CustomTable method + will expect a defined schema to be provided (good best practice, in any case). """ + +from llmware.resources import CustomTable + + +def building_custom_table_from_json(config_option=2): + + # point fp and fn at the file_path of the JSON file + fp = "/local/path/to/json_file" + + # note: this example uses the "model_list.json" example file found in the examples repository + # -- if substituting for your own json/jsonl, please also adjust the sample query below + fn = "model_list.json" + + # first analyze the csv and confirm that the rows and columns are consistently being extracted + # key_list is optional and can be removed - will validate all columns + # if key_list provided, then will look only at the keys provided + analysis = CustomTable().validate_json(fp,fn, key_list=["model_name", "context_window"]) + + print("\nAnalysis of the JSON file") + for key, value in analysis.items(): + print(f"analysis: {key} - {value}") + + if not (1 <= config_option <= 4): + print("\nsetting config to default == 1") + config_option = 1 + + table_name = "model_table_100" + db_name = "mongo" + output = None + + # loading a json into a database has three main steps + # 1. construct CustomTable object + # 2. load_json - *** where most of the configuration will occur *** + # 3. insert_rows + + ct = CustomTable(db=db_name,table_name=table_name) + + if config_option == 1: + + # load_json - Option #1 - this is the simplest case + # -- will use the first row as the 'test row' to extract keys for the schema + # -- will infer the data type for each column as either 'text' | 'integer' | 'float' + + output = ct.load_json(fp ,fn) + + elif config_option == 2: + + # load_json - Option #2 - pass a subset of the keys to use for the schema + # -- will look for key in each row, and ignore keys not in the selected_keys + + target_keys = ["model_name", "context_window"] + output = ct.load_json(fp, fn, selected_keys=target_keys) + + elif config_option == 3: + + # load_csv - Option #3 - pass an explicit data type mapping, for all or some columns, + # which will 'over-ride' the estimation + # -- for json, the mapping needs to be done by the name of the key, e.g., "context_window" + + dt_mapping = {"context_window": "decimal"} + output = ct.load_json(fp, fn, data_type_map=dt_mapping) + + elif config_option == 4: + + # load_csv - Option #4 - pass a complete schema to be used + + # note: this option is not intended for use with the customer_table.csv example + my_schema = {"model_name": "text", "model_family": "text", "context_window": "text"} + output = ct.load_json(fp, fn, schema=my_schema) + + print("\nLoad CSV output") + for key, value in output.items(): + print(f"output: {key} - {value}") + + # spot-check the rows that have been created before inserting into database as a final check + print("\nSpot-Check Rows Before Inserting into DB Table") + sample_size = min(len(ct.rows), 10) + for x in range(0 ,sample_size): + print("rows: ", x, ct.rows[x]) + + # when ready, uncomment, and insert the rows into the DB + ct.insert_rows() + + # basic query + customer = ct.lookup("model_name", "slim-extract-tool") + print("\nLookup from DB") + print(f"customer_record: ", customer) + + return 0 + + +if __name__ == "__main__": + + building_custom_table_from_json(config_option=2) diff --git a/solutions/sources/lookup_docs_and_blocks.py b/solutions/sources/lookup_docs_and_blocks.py new file mode 100644 index 0000000..ab28c6c --- /dev/null +++ b/solutions/sources/lookup_docs_and_blocks.py @@ -0,0 +1,54 @@ + +""" This example demonstrates how to 'lookup' and retrieve an entire document, or a selected block, from the + text collection DB. """ + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def create_agreements_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "Agreements") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +def doc_lookup(library): + + print(f"\nExample - Retrieving by Document") + + # create a Query instance + q = Query(library) + + # step 1 - pull all of the blocks for a particular document + # -- use either "doc_id" number or "file_source" file name + + my_doc = q.document_lookup(file_source="Nyx EXECUTIVE EMPLOYMENT AGREEMENT.pdf") + # my_doc = q.document_lookup(doc_id = 1) + + for i, result in enumerate(my_doc): + print(f"result - {i} - {result['file_source']} - block - {result['block_ID']} - page - {result['page_num']} - text - {result['text']}") + + # step 2 - to lookup a specific block in a document + my_block = q.block_lookup(block_id=0, doc_id=1) + + print(f"\nExample - Selecting a specific block") + print("my block: ", my_block) + + return my_block + + +if __name__ == "__main__": + + # use any supported database - mongo, postgres or sqlite + LLMWareConfig().set_active_db("mongo") + + lib = create_agreements_sample_library("doc_and_blocks_lookup_example") + my_results = doc_lookup (lib) + diff --git a/solutions/sources/model_list.json b/solutions/sources/model_list.json new file mode 100644 index 0000000..e34ccea --- /dev/null +++ b/solutions/sources/model_list.json @@ -0,0 +1,62 @@ +[ + { + "model_name": "slim-extract", + "model_family": "HFGenerativeModel", + "context_window": 2048, + "description": "specialized custom extraction" + }, + { + "model_name": "slim-extract-tool", + "model_family": "GGUFGenerativeModel", + "context_window": 2048, + "description": "specialized custom extraction - quantized for local inference" + }, + { + "model_name": "industry-bert-contracts", + "model_family": "HFEmbeddingModel", + "context_window": 512, + "description": "embedding model fine-tuned on contracts" + }, + { + "model_name": "slim-summary-tool", + "model_family": "GGUFGenerativeModel", + "context_window": 2048, + "description": "specialized summary function model - quantized for local inference" + }, + { + "model_name": "slim-topics", + "model_family": "HFGenerativeModel", + "context_window": 2048, + "description": "specialized topic categorization function model" + }, + { + "model_name": "slim-boolean-tool", + "model_family": "GGUFGenerativeModel", + "context_window": 2048, + "description": "specialized quantized model for boolean question answering with explanation capability" + }, + { + "model_name": "bling-answer-tool", + "model_family": "GGUFGenerativeModel", + "context_window": 2048, + "description": "general purpose quantized question-answering model - small and fast" + }, + { + "model_name": "text-davinci-003", + "model_family": "OpenAIGenModel", + "context_window": 4096, + "description": "OG OpenAI model that started the GenAI boom in 2022" + }, + { + "model_name": "llama-2-chat", + "model_family": "GGUFGenerativeModel", + "context_window": 4096, + "description": "popular 7b chat model quantized for local deployment" + }, + { + "model_name": "slim-sql-tool", + "model_family": "GGUFGenerativeModel", + "context_window": 2048, + "description": "specialized small text2sql quantized model for local deployment" + } +] \ No newline at end of file diff --git a/solutions/sources/ocr_embedded_doc_images.py b/solutions/sources/ocr_embedded_doc_images.py new file mode 100644 index 0000000..f6519e5 --- /dev/null +++ b/solutions/sources/ocr_embedded_doc_images.py @@ -0,0 +1,173 @@ + +""" Example: ** Applying OCR to Images in LLMWare Library ** + + This example shows how to: + + A. identify images in a library (post the initial parsing) + B. run an OCR against the images to derive the text from the image using the OCR + C. insert the text into the database library collection for subsequent retrieval. + + Note: this example uses additional python dependencies: + + -- pip3 install pytesseract + + Note: this example uses an OCR engine, which is outside of the core llmware package. To install on Ubuntu: + + -- sudo apt install tesseract-ocr + -- sudo apt install libtesseract-dev + + [Other platforms: + -- Mac: brew install tesseract + -- Windows: GUI download installer - see UB-Mannheim @ www.github.com/UB-Mannheim/tesseract/wiki + + Running this script will NOT make any changes to the original "image" block record in the text collection. + Rather than update/replace the existing record, the script will create a new supplemental entry for each 'image'. + + Each new record will have the following attributes: + + --"text" block with the text derived from the OCR, including the original source doc_ID, file source name and + page number for easy reference + + --new block_ID starting at 100000 (safely out of the 'block namespace' of the original document, and easy to + identify as 'derived' text from an OCR, rather than an original part of the document + + --text chunking applied to the OCR output, especially useful if it is a large image with a lot of text, e.g., + a scanned page of a book - if the image contains a large text passage, it will be chunked and saved as + potentially several individual text blocks, 'chunked' according to the text chunk size parameter. + + --a custom 'string' flag in special_field1 that indicates the text was created by an OCR, and includes a + reference to the original doc_ID and block_ID + + --optional threshold for length of OCR to text to capture, e.g., if <10 characters captured, then may be + preferable to skip (higher probability that image is noisy) + +""" + + +from llmware.library import Library +from llmware.configs import LLMWareConfig +from llmware.resources import CollectionRetrieval, CollectionWriter +from llmware.parsers import ImageParser + +from importlib import util +if not util.find_spec("pytesseract"): + print("\nto run this example requires additional dependencies, including pytesseract - see comments above in " + "this script. to install pytesseract: pip3 install pytesseract.") + + +def ocr_images_in_library(library_name, add_new_text_block=False, chunk_size=400, min_chars=10): + + lib = Library().load_library(library_name) + image_path = lib.image_path + + # check here to see the images extracted from the original parsing + print("update: image source file path: ", image_path) + + # query the collection DB by content_type == "image" + image_blocks = CollectionRetrieval(library_name).filter_by_key("content_type", "image") + doc_update_list = {} + new_text_created = 0 + + # iterate through the image blocks found + for i, block in enumerate(image_blocks): + + # "external_files" points to the image name that will be found in the image_path above for the library + img_name = block["external_files"] + + # each doc_ID is unique for the library collection + doc_id = block["doc_ID"] + + # block_IDs are unique only for the document, and generally run in sequential ascending order + block_id = block["block_ID"] + + # note: _id not used, but it is a good lookup key that can be easily inserted in special_field1 below + bid = block["_id"] + + # preserve_spacing == True will keep \n \r \t and other white space + # preserve_spacing == False collapses the white space into a single space for 'more dense' text only + output = ImageParser(text_chunk_size=chunk_size).process_ocr(image_path,img_name,preserve_spacing=False) + + print("update: ocr output: ", output) + + # note: test before writing to the collection + if add_new_text_block: + + for text_chunk in output: + + if text_chunk.strip(): + + # optional to keep only more substantial chunks of text + if len(text_chunk) > min_chars: + + # ad hoc tracker to keep incrementing the block_id for every new image in a particular doc + if doc_id in doc_update_list: + new_block_id = doc_update_list[doc_id] + doc_update_list.update({doc_id: new_block_id+1}) + else: + new_block_id = 100000 + doc_update_list.update({doc_id: new_block_id+1}) + + new_block = block + + # feel free to adapt these attributes to fit for purpose + new_block.update({"block_ID": new_block_id}) + new_block.update({"content_type": "text"}) + new_block.update({"embedding_flags": {}}) + new_block.update({"text_search": text_chunk}) + new_block.update({"special_field1": f"OCR applied to image in document - " + f"{doc_id} - block - {block_id}"}) + + # new _id will be assigned by the database directly + if "_id" in new_block: + del new_block["_id"] + + print("update: writing new text block - ", new_text_created, doc_id, block_id, text_chunk, new_block) + + # creates the new record + CollectionWriter(ln).write_new_parsing_record(new_block) + + new_text_created += 1 + + return new_text_created + + +# main execution script starts here + +if __name__ == "__main__": + + # select collection db - mongo, sqlite, or postgres + LLMWareConfig().set_active_db("postgres") + + # create a library (source documents must have embedded images!) + ln = "my_library" + fp = "/path/to/pdf_or_office_files_with_images/" + lib = Library().create_new_library(ln) + + # parse the documents + lib.add_files(fp, get_images=True,get_tables=True, chunk_size=400, max_chunk_size=600, + smart_chunking=1, verbose_level=2) + + print("done parsing") + + # runs ocr on the images in the newly-created library + # set add_new_text_block == True to add new rows in the database (otherwise, will just run the OCR in memory) + new_blocks = ocr_images_in_library(ln,add_new_text_block=False,chunk_size=400, min_chars=10) + + print("done with ocr processing") + + g = CollectionRetrieval(ln).get_whole_collection() + + # may need to loop through the iterator if extremely large collection (otherwise, pull_all into memory OK) + + blocks = g.pull_all() + + ocr_count = 0 + + # look at the new text entries created and inserted into the collection + for i, b in enumerate(blocks): + if b["block_ID"] > 9999: + print("update: new ocr text entry: ", ocr_count, b["doc_ID"], b["block_ID"], b["content_type"], + b["special_field1"], b["text_search"]) + + ocr_count += 1 + diff --git a/solutions/sources/office_parser_new_configs.py b/solutions/sources/office_parser_new_configs.py new file mode 100644 index 0000000..383e53a --- /dev/null +++ b/solutions/sources/office_parser_new_configs.py @@ -0,0 +1,86 @@ + +""" Starting with llmware 0.2.8, new configuration options are exposed in the Office parser, with many more to come. + This should not result in any breaking changes in existing code, but only exposes new configuration options. + + Note: the configuration options for the Office parser apply to parsing of the following document types: + + -- Word Documents -- .docx + -- Powerpoints -- .pptx + -- Excel -- .xlsx + + Wherever possible, the configuration options match the PDF parsing configuration options. """ + +from llmware.library import Library + +fp = "/path/to/my/office/files" + +lib = Library().create_new_library("my_library") + +# standard call to 'ingest' files into a library (implicitly calls Parser and manages the details) +lib.add_files(input_folder_path=fp) + +# new configuration options in Parser class, exposed in .add_files method for convenience, although it can +# always be accessed by direct construction of a Parser + +# CHUNK SIZING - implemented in both PDF and Office parser +# Measured in string len characters, e.g., 400 characters as target text chunk size, with max of 600 characters + +# -- there are 4 text chunking strategies supported in the Office parser: +# -- smart_chunking = 0 -> will stop at the target chunk size (or as close as possible) - breaks words +# -- smart_chunking = 1 -> will stop at the first white space after the target chunk size - preserves words +# -- smart_chunking = 2 -> will look for a natural break, either a "." or "\r" or "\n", up to the max size +# -- smart_chunking = 3 -> will follow the structures as set out in the Office XML document + +# -- note: this applies to all text content types, but does not apply to tables, which will be parsed and captured +# separately as "table" content types - and generally table will be kept in a single logical chunk, which may +# exceed the limits of the text chunking. + +# -- note: it is possible that the char lengths will vary by 1-3 bytes, as there are safeguards to prevent breaking a +# multi-byte utf-8 character. If there are any breaking changes, they will generally be remediated before +# entering the text in the database automatically. + +lib.add_files(input_folder_path=fp, chunk_size=400, max_chunk_size=600, smart_chunking=1) + +# CAPTURE PARAMETERS + +# -- ability to turn on/off the capture of other features in the parser, e.g, whether to capture images, tables, +# and formatted header text (e.g., BOLD, ITALICS and Large Font) + +# -- if you are only interested in text, then you can turn off these features for faster performance + +lib.add_files(input_folder_path=fp, get_images=False, get_tables=False, get_header_text=False) + +# ENCODING + +# -- encoding = "utf-8" - new default - supports Western European characters, and many characters across the +# wider Unicode set of code points - note: not full support yet for Asian languages and characters + +# -- right now, the Office parser only supports UTF-8 encoding +# -- will explore options to add ASCII 7-bit range in the future +# -- if there is a problem parsing the text's UTF-8 character set, then the problematic text is converted to ASCII. + +lib.add_files(input_folder_path=fp, encoding="utf-8") + +# COPY FILES TO LIBRARY + +# -- by default, input files are collated to different parsers, and then at the end of the process, copied to a +# library file structure for convenient future access by the library and for downstream retrieval applications +# that query the library, and then want to provide direct access to the files. + +# -- if you do not want to duplicate files, then set copy_files_to_library = False + +lib.add_files(input_folder_path=fp, copy_files_to_library=False) + +# VERBOSE OUTPUT + +# -- four modes of output to screen +# -- verbose_level == 0 -> suppresses virtually all output, except major errors/problems +# -- verbose_level == 1 -> displays 1st ten pages of document text in text chunks as parsing +# -- verbose_level == 2 -> displays file name being parsed only +# -- verbose_level == 3 -> deep debugging mode (not recommended) - useful for llmware dev team in tracing errors + +lib.add_files(input_folder_path=fp, verbose_level=2) + + + + diff --git a/solutions/sources/page_number_filter.py b/solutions/sources/page_number_filter.py new file mode 100644 index 0000000..3120959 --- /dev/null +++ b/solutions/sources/page_number_filter.py @@ -0,0 +1,60 @@ + +"""This example demonstrates how to use a page number filter to narrow search retrievals by specific + documents and pages - using document_filter and page_lookup methods in the Query class. """ + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def create_agreements_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "Agreements") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +def page_lookup(library): + + print(f"\nExample - Retrieving by Document and Page Number") + + # create a Query instance + q = Query(library) + + # step 1 - select a set of documents from the library + # -- this can be accomplished in several ways, e.g., + # (A) doc_id_list = [1,2,3,4,5] -> will select documents with doc_ID = 1,2,3,4,5 + # (B) run a document filter query, e.g., doc_id_list = Query(library).document_filter("my_query") + # (C) leave blank -> will retrieve all of the doc id from the library + + # in this case, will run a document_filter query to generate a doc_id_list with contracts with the name "apollo" + my_query = "apollo" + doc_id_list = q.document_filter(my_query) + + print(f"\nstep 1 - build doc_id_list with query - {my_query} - ", doc_id_list) + + # define a target page list - useful for template documents with key names/phrases in specific places + page_list = [1,12] + + print(f"step 2 - execute page lookup for pages - {page_list}\n") + + query_results = q.page_lookup(page_list=page_list, doc_id_list=doc_id_list) + + for i, result in enumerate(query_results): + print(f"result - {i} - {result['file_source']} - page - {result['page_num']} - text - {result['text']}") + + return query_results + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + lib = create_agreements_sample_library("lib_page_filter_example") + my_results = page_lookup (lib) + diff --git a/solutions/sources/parse_csv_custom.py b/solutions/sources/parse_csv_custom.py new file mode 100644 index 0000000..6ecac10 --- /dev/null +++ b/solutions/sources/parse_csv_custom.py @@ -0,0 +1,173 @@ + +""" This script illustrates options for parsing csv and tsv files into a Library in LLMWare, including methods for + mapping custom configured csv or tsv file, which is intended for use with 'pseudo-db' CSV files + + Option #1 - Standard CSV/TSV Parsing + + -- when using a bulk ingest Parsing method, the parser will route csv files to the 'standard' + TextParser - which will look to extract and 'aggregate' the text from the csv as source content, and + add to "text" and/or "table" attributes -> no "structure" information or targeted keys are captured + + -- the standard TextParser() is designed for ad hoc extraction of text content + + -- if file is csv, then by default delimiter assumed to be ',' (which can be adjusted) + -- if file is tsv, then by default delimited assumed to be '\t' (which can be adjusted) + + -- this is illustrated in example 1 and example 2 below. + + Option #2 - Custom Configured CSV/TSV Parsing + + -- if the CSV file is a pseudo-db with structured attributes, then it can be configured using a special + parsing method, as outlined below in examples 3 and 4. + + -- the benefit of this custom mapping is that key column attributes will be saved as "metadata" in addition + to the option to provide a custom over-write of doc_ID and block_ID parameters for indexing of text + chunks in the database + + """ + +from llmware.parsers import Parser, TextParser +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + +import time +import ast + +# All three text databases supported (mongo, postgres, and sqlite) +# if it is highly varied unstructured content, we would recommend Mongo given its flexibility +# if any validation errors with Postgres or SQLite, then we would recommend either further preprocessing the csv or +# ... trying with Mongo + +LLMWareConfig().set_active_db("mongo") + + +def standard_csv_parsing(fp, fn, delimiter=","): + + """ Example #1 - This example shows the 'standard' text handler for csv """ + + # the standard csv parser will interpret the output as a table, trying to preserve the 'row-by-row' and + # 'cell-by-cell' structure (without any keys/labels). There are several options how the output text will + # be aggregated and saved as a single 'text' or 'table' entry: + + # --batch_size: this is the number of rows that will be aggregated into each text entry + # e.g., if batch_size == 1, then each row will be a single entry in the database + # if batch_size == 10, then 10 rows will be aggregated into a single 'text' entry in the db + + # --interpret_as_table: + # if true, then text will be packaged as a string wrapping a nested list. + # if false, then text will be packaged as a single text stream separated by "\t" between entries + + # --optional parameters allow configuration of encoding ('utf-8-sig'), errors ('ignore'), + # and separator ("\n") (applied at end of each row) + + # to experiment with the expected output, try the method below, which will not write to the DB, but outputs + # a list in memory + + output = TextParser().csv_file_handler(fp,fn,interpret_as_table=False,batch_size=1, delimiter=delimiter) + + return output + + +def standard_csv_parsing_into_library(fp, library_name): + + """ Example #2 - building on the first example, this example will parse a set of 'standard' CSV files + directly into the library - if file type is 'tsv' then delimiter automatically applied as '\t' """ + + # create new library + lib = Library().create_new_library(library_name) + + # create parser object, and pass the library to use to write the parsing output + parser = Parser(lib) + + # directly call the parse_text method, which will parse text files (csv, tsv, json, jsonl, txt, md) + # this parsed output will be saved to the database by default + + output = parser.parse_text(fp, interpret_as_table=False,batch_size=1,delimiter=",", encoding="utf-8-sig", + write_to_db=True) + + return output + + +def configured_csv_parsing(fp, fn,library_name): + + """ Example #3 - This example shows how to use mappings for a customized csv """ + + # metadata is a dictionary mapping of key names to columns in the csv file + # the 'keys' correspond to the keys that will be added to the library + # the 'values' correspond to the columns found in the source CSV (starting with 0 index) + + # metadata map must have "text" mapping + # if "doc_ID" or "block_ID" mapping provided, then will "over-write" the default doc_ID and block_ID and + # use the mapping provided in the source CSV + + # for all other attributes (e.g., not text, doc_ID, block_ID), the keys will be stored in "special_field1" of + # the database. For Mongo, the keys will be stored directly as a dictionary, while for Postgres and SQLite, + # it will be stored as text string, which must be converted upon use back into a dictionary (see below for + # retrieval example) + + # step 1 - create metadata mapping, + # e.g., number indexes map to columns in the csv, 0-index and negative slicing supported (-1 is last column) + metadata = {"text": -1, "doc_ID": 0, "key1": 1, "key2": 2, "key3": 3, "key4": 4} + columns = 6 + + # step 2 - create library + lib = Library().create_new_library(library_name) + parser = Parser(lib) + + # step 3 - invoke parse_csv_config method + # -- note: if file is not comma delimited, then set delimiter + # -- if file is tab delimited, e.g. tsv, then delimiter = "\t" + + print("step 1 - parsing") + t0 = time.time() + parser_output = parser.parse_csv_config(fp, fn, cols=columns, mapping_dict=metadata,delimiter=",") + print(f"done parsing - time - {time.time() - t0} - summary - {parser_output}") + + return parser_output + + +def example4_run_query_configured_input(library_name=None, query=""): + + """ Example #4 - once the custom csv/tsv is parsed into a Library, it can be used like any other content with the + additional attributes available in special_field1- which can be retrieved as demonstrated below. + + -- note: the example below illustrates a 'text_query' but will apply exactly the same for a 'semantic_query' + + """ + + # run query + lib = Library().load_library(library_name) + + q = Query(lib).text_query(query) + + for j, results in enumerate(q): + + meta = "" + doc_id = -1 + + # the metadata attributes are saved in the database under "special_field1" column + if "special_field1" in results: + meta = results["special_field1"] + if isinstance(meta, str): + try: + meta = ast.literal_eval(meta) + except: + print(f"could not convert meta string back into dictionary - {meta}") + + if "doc_ID" in results: + doc_id = results["doc_ID"] + + text = results["text"] + + if len(text) > 200: + text = text[0:200] + + print(f"\nresults - {j} - query - {query}") + print(f"results - text - {text}") + print(f"results - doc_ID - {doc_id} - metadata - {meta}") + + print("done") + + return 0 + diff --git a/solutions/sources/parse_documents.py b/solutions/sources/parse_documents.py new file mode 100644 index 0000000..dcd087d --- /dev/null +++ b/solutions/sources/parse_documents.py @@ -0,0 +1,72 @@ + +"""This 'Getting Started' example demonstrates how to parse document files into a library + 1. Create a library + 2. Assemble input files into a single folder path (fp) + 3. Pass the folder path to library.add_files(fp) to automatically parse, text chunk, index +""" + +import os +from llmware.library import Library +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def parsing_documents_into_library(library_name, sample_folder): + + print(f"\nExample - Parsing Files into Library") + + # create new library + print (f"\nStep 1 - creating library {library_name}") + library = Library().create_new_library(library_name) + + # load the llmware sample files + # -- note: if you have used this example previously, UN-Resolutions-500 is new path + # -- to pull updated sample files, set: 'over_write=True' + + sample_files_path = Setup().load_sample_files(over_write=False) + print (f"Step 2 - loading the llmware sample files and saving at: {sample_files_path}") + + # note: to replace with your own documents, just point to a local folder path with the documents + ingestion_folder_path = os.path.join(sample_files_path, sample_folder) + + print (f"Step 3 - parsing and indexing files from {ingestion_folder_path}") + + # add files is the key ingestion method - parses, text chunks and indexes all files in folder + # --will automatically route to correct parser based on file extension + # --supported file extensions: .pdf, .pptx, .docx, .xlsx, .csv, .md, .txt, .json, .wav, and .zip, .jpg, .png + + parsing_output = library.add_files(ingestion_folder_path) + + print (f"Step 4 - completed parsing - {parsing_output}") + + # check the updated library card + updated_library_card = library.get_library_card() + doc_count = updated_library_card["documents"] + block_count = updated_library_card["blocks"] + print(f"Step 5 - updated library card - documents - {doc_count} - blocks - {block_count} - {updated_library_card}") + + # check the main folder structure created for the library - check /images to find extracted images + library_path = library.library_main_path + print(f"Step 6 - library artifacts - including extracted images - saved at folder path - {library_path}") + + return parsing_output + + +if __name__ == "__main__": + + # note on sample documents - downloaded by Setup() + # UN-Resolutions-500 is 500 pdf documents + # Invoices is 40 pdf invoice samples + # Agreements is ~15 contract documents + # AgreementsLarge is ~80 contract documents + # FinDocs is ~15 financial annual reports and earnings + # SmallLibrary is a mix of ~10 pdf and office documents + + # this is a list of document folders that will be pulled down by calling Setup() + sample_folders = ["Agreements", "Invoices", "UN-Resolutions-500", "SmallLibrary", "FinDocs", "AgreementsLarge"] + + LLMWareConfig().set_active_db("sqlite") + + library_name = "parsing_test_lib" + selected_folder = sample_folders[0] + output = parsing_documents_into_library(library_name, selected_folder) \ No newline at end of file diff --git a/solutions/sources/parse_images.py b/solutions/sources/parse_images.py new file mode 100644 index 0000000..fab95cb --- /dev/null +++ b/solutions/sources/parse_images.py @@ -0,0 +1,52 @@ + +""" This example demonstrates how to parse images using OCR into a Library + 1. This uses the tesseract OCR engine - which must be loaded as a prerequisite native lib + 2. This illustrates how to explicitly parse an image file (png, jpeg) + 3. If image files are included in an overall ".add_files", then they will be handled as images automatically +""" + +import os +from llmware.library import Library, LibraryCatalog +from llmware.parsers import Parser, ImageParser +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +# Demonstrate adding files to a library, which implicitly parses them and creates blocks +def parsing_image_sources_into_library(library_name): + + print(f"Example - Parsing Images") + + # load the llmware sample files + # -- to pull updated sample files, set: 'over_write=True' + + sample_files_path = Setup().load_sample_files(over_write=False) + print (f"Step 2 - loading the llmware sample files and saving at: {sample_files_path}") + + # note: to replace with your own documents, just point to a local folder path with the documents + # Perform OCR to extract text from iamges (e.g scanned documents) + image_file_path = os.path.join(sample_files_path, "Images") + + parser = Parser() + + for i, images in enumerate(os.listdir(image_file_path)): + + # image_file = "Apache2_License.png" + print(f"\n > Parsing {images}") + image_parsed_output = parser.parse_one_image(image_file_path, images,save_history=True) + block_text = image_parsed_output[0]["text"] + print(f"\nFirst block found in image:\n{block_text}") + + print(f"\nText Blocks extracted in total: ", len(parser.parser_output)) + + return 0 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + lib_name = "my_image_library_23" + + x = parsing_image_sources_into_library(lib_name) + diff --git a/solutions/sources/parse_in_memory.py b/solutions/sources/parse_in_memory.py new file mode 100644 index 0000000..db523fb --- /dev/null +++ b/solutions/sources/parse_in_memory.py @@ -0,0 +1,68 @@ + +"""This example demonstrates how to parse individual documents into memory + -- Does not require the use of database +""" + +import os +from llmware.parsers import Parser +from llmware.setup import Setup + + +def parsing_files_into_memory(folder_name): + + print(f"\nExample - Parsing Documents into Memory (no DB required)") + + # Load the llmware sample files + print (f"\nstep 1 - loading the llmware sample files") + sample_files_path = Setup().load_sample_files() + print (f"step 2- llmware sample files saved locally at: {sample_files_path}") + + # Parse individual documents. The output will be a list of blocks (dicts with metadata) + ingestion_file_path = os.path.join(sample_files_path,folder_name) + + files = os.listdir(ingestion_file_path) + + print(f"\t\t--total of {len(files)} documents found in folder path - will parse each one.") + + for i, doc in enumerate(files): + + parser_output = Parser().parse_one(ingestion_file_path,doc,save_history=False) + + if parser_output: + print(f"\nupdate: parsed document - {i} - {doc}") + print(f"update: extracted {len(parser_output)} blocks of information - showing first 10 blocks for display:") + + if len(parser_output) < 10: + sample_display_len = len(parser_output) + else: + sample_display_len = 10 + + parser_output = parser_output[0:sample_display_len] + + for j, blocks in enumerate(parser_output): + text = blocks["text"] + if len(text) > 100: + text = text[0:100] + " ... " + print(f"update: block - {blocks['block_ID']} - {text}") + else: + print(f"\nupdate: did not find any content for file - {doc}") + + return 0 + + +if __name__ == "__main__": + + # note on sample documents - downloaded by Setup() + # UN-Resolutions-500 is 500 pdf documents + # Invoices is 40 pdf invoice samples + # Agreements is ~15 contract documents + # AgreementsLarge is ~80 contract documents + # FinDocs is ~15 financial annual reports and earnings + # SmallLibrary is a mix of ~10 pdf and office documents + + # this is a list of document folders that will be pulled down by calling Setup() + sample_folders = ["Agreements", "Invoices", "UN-Resolutions-500", "SmallLibrary", "FinDocs", "AgreementsLarge"] + + library_name = "parsing_test_lib_0" + selected_folder = sample_folders[0] + output = parsing_files_into_memory (selected_folder) diff --git a/solutions/sources/parse_into_prompt.py b/solutions/sources/parse_into_prompt.py new file mode 100644 index 0000000..f004428 --- /dev/null +++ b/solutions/sources/parse_into_prompt.py @@ -0,0 +1,74 @@ + +""" This example demonstrates how to parse a document 'in-flight' as part of a Prompt using "Prompt with Sources" + + 1. Load sample documents + 2. Create a Prompt object + 3. Load a locally-run BLING model (may take a few minutes to download the first time from HuggingFace) + 4. Add Document as Source to Prompt + -- this will automatically parse the source document, text chunk, and package into prompt context window + -- optional query filter to narrow the list of text chunks packaged into the source + 5. Invoke .prompt_with_sources method + -- this will take the packaged source, and run inference on the LLM +""" + + +import os + +from llmware.prompts import Prompt +from llmware.setup import Setup + + +def prompt_source (model_name): + + print(f"\nExample: Parse and Filter Documents Directly in Prompt to LLM") + + # load the llmware sample files + print (f"\nstep 1 - loading the llmware sample files") + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path,"Agreements") + + # load bling model which will be used for the inference (will run on local laptop CPU) + # --note: typically requires 16 GB laptop RAM + print (f"step 2 - loading model {model_name}") + + # create prompt object + prompter = Prompt() + prompter.load_model(model_name) + + # this is the question that we will ask to each document + research = {"topic": "base salary", "prompt": "What is the executive's base salary?"} + + for i, contract in enumerate(os.listdir(contracts_path)): + + # (optional) safety check to exclude Mac-specific file artifact + if contract != ".DS_Store": + + print("\nAnalyzing Contract - ", str( i +1), contract) + print("Question: ", research["prompt"]) + + # contract is parsed, text-chunked, and then filtered by "base salary' + # --note: query is optional - if no query, then entire document will be returned and added as source + source = prompter.add_source_document(contracts_path, contract, query=research["topic"]) + + # take a look at the created source + print("Source created from document: ", source) + + # calling the LLM with 'source' information from the contract automatically packaged into the prompt + responses = prompter.prompt_with_source(research["prompt"], prompt_name="default_with_context", temperature=0.3) + + for r, response in enumerate(responses): + print("\nLLM Response: ", response["llm_response"]) + + # We're done with this contract, clear the source from the prompt + # -- note: if looking to aggregate or keep 'running' source, then do not clear + prompter.clear_source_materials() + + return 0 + + +if __name__ == "__main__": + + model_name = "llmware/bling-1b-0.1" + prompt_source(model_name) + + diff --git a/solutions/sources/parse_jsonl_custom.py b/solutions/sources/parse_jsonl_custom.py new file mode 100644 index 0000000..367c278 --- /dev/null +++ b/solutions/sources/parse_jsonl_custom.py @@ -0,0 +1,148 @@ + +""" This script illustrates options for parsing JSON and JSONL files into a Library in LLMWare, including the ability + to provide a custom configured mapping intended for use with 'pseudo-db' structured JSON and JSONL files + + Option # 1- Standard JSON/JSON parsing - + + -- when using a bulk ingest Parsing method, the parser will route json and jsonl files to the 'standard' + TextParser - which will look for a "text" key in the JSON/JSONL to extract as the intended text + + -- if no 'text' key found, then the parse will return an empty output list [] + + -- you can provide an optional key_list parameter to TextParser, which will then by default capture the + selected fields and aggregate to form the text input, e.g., + + TextParser().jsonl_file_handler(fp,fn, key_list=["context", "source", "ID"] + + ... where "context", "source" and "ID" represent keys found in the source json/jsonl file + + -- the standard TextParser() is designed for ad hoc extraction of text content, not to preserve the keys + + -- use of this method is shown in the first example below + + Option # 2- Custom Configured JSON/JSONL parsing - + + -- in addition to the standard parsing method, there is the ability to customize the mappings for a + JSON/JSONL file in which the keys are intended to be used for follow-up lookup and retrieval + + -- Parser().parse_json_config method - this is shown in the second and third examples below + + """ + + +from llmware.parsers import Parser, TextParser +from llmware.library import Library +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + +import time +import ast + +# All three text databases supported (mongo, postgres, and sqlite) +# if it is highly varied unstructured content, we would recommend Mongo given its flexibility +# if any validation errors with Postgres or SQLite, then we would recommend either preprocessing the json or +# ... trying with Mongo + +LLMWareConfig().set_active_db("mongo") + + +def standard_json_parsing(fp, fn): + + """ This example shows the 'standard' text handler for json/jsonl """ + + # the selected keys should map to dictionary keys found in the JSON/JSONL + # if no keys passed, then by default, parser will only look for a "text" key + # the parser objective is extracting/aggregating the content of the file, not using the 'structure' of the keys + # if interpret_as_table is True, then returns each row as a LIST of elements, corresponding to the selected keys + # if interpret_as_table is False, then returns a text string, which is concatenation of the text found in each + # key, and will use the value of the separator to combine each key, + # e.g., value1 + separator + value2 + separator ... + + selected_keys = ["key1", "key2", "key3"] # e.g., "context", "source", "ID" or other keys in json + + output = TextParser().jsonl_file_handler(fp,fn,key_list=selected_keys,interpret_as_table=False,separator="\n") + + return output + + +def configured_json_parsing(fp, fn, library_name): + + """ This example shows how to use mappings for a customized json/jsonl """ + + # metadata is a dictionary mapping of key names to keys in the json file + # the 'keys' correspond to the keys that will be added to the library + # the 'values' correspond to the keys that will be found in the JSON/JSONL source file + + # metadata must have "text" mapping + # if "doc_ID" or "block_ID" mapping provided, then will "over-write" the default doc_ID and block_ID and + # use the mapping provided in the source JSON/JSONL + + # for all other attributes (e.g., not text, doc_ID, block_ID), the keys will be stored in "special_field1" of + # the database. For Mongo, the keys will be stored directly as a dictionary, while for Postgres and SQLite, + # it will be stored as text string, which must be converted upon use back into a dictionary (see below for + # retrieval example) + + # step 1 - create metadata mapping + # -- must have 'text' key mapped to key in json source + # -- all other keys are 'optional' and can be any number from 0 - N + # -- generally, key2, etc. should map to the name of the key in the JSON file, although you are free to re-name + + metadata= {"text": "json_source_key_mapping_to_main_text_input", + "key2": "json_source_key2", + "key3": "key3"} + + # step 2 - create new library + lib = Library().create_new_library(library_name) + parser = Parser(lib) + + # step 3 - invoke parse_json_config method + print("step 1 - parsing") + t0 = time.time() + parser_output = parser.parse_json_config(fp, fn, mapping_dict=metadata) + print(f"done parsing - time - {time.time() - t0} - summary - {parser_output}") + + return parser_output + + +def run_query_configured_input (library_name=None,query=""): + + """ Once the custom json/jsonl is parsed into a Library, it can be used like any other content with the + additional json/jsonl attributes available in special_field1- which can be retrieved as demonstrated below. + + -- note: the example below illustrates a 'text_query' but will apply exactly the same for a 'semantic_query' + """ + + # run query + lib = Library().load_library(library_name) + + q = Query(lib).text_query(query) + + for j, results in enumerate(q): + + meta = "" + doc_id = -1 + + # the mapped keys from the json file are all stored in "special_field1" of the library dictionary entry + # and can be retrieved as a string that can be mapped back into a dictionary as outlined below + + if "special_field1" in results: + meta = results["special_field1"] + if isinstance(meta,str): + try: + meta = ast.literal_eval(meta) + except: + print(f"could not convert meta string back into dictionary - {meta}") + + if "doc_ID" in results: + doc_id = results["doc_ID"] + + text = results["text"] + + print(f"\nresults - {j} - query - {query}") + print(f"results - text - {text}") + print(f"results - doc_ID - {doc_id} - metadata - {meta}") + + print("done") + + return 0 + diff --git a/solutions/sources/parse_pdf_by_ocr.py b/solutions/sources/parse_pdf_by_ocr.py new file mode 100644 index 0000000..1ab12ef --- /dev/null +++ b/solutions/sources/parse_pdf_by_ocr.py @@ -0,0 +1,75 @@ + +""" This example demonstrates how to parse PDF documents consisting of scanned pages using OCR + + Parsing a PDF-by-OCR is much slower and loses metadata, compared with a digital parse - but this is a + necessary fall-back for many 'paper-scanned' PDFs, or in the relatively rare cases in which + digital parsing is not successful + + NOTE: there are several dependencies that must be installed to run this example: + + pip install: + -- pip3 install pytesseract + -- pip3 install pdf2image + + core libraries: + -- tesseract: e.g., (Mac OS) - brew install tesseract or (Linux) - sudo apt install tesseract + -- poppler: e.g., (Mac OS) - brew install poppler or (Linux) - sudo apt-get install -y poppler-utils + for Windows download see - https://poppler.freedesktop.org/ + +""" + +import os +import time + +from llmware.parsers import Parser +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + +from importlib import util +if not util.find_spec("pytesseract") or not util.find_spec("pdf2image"): + print("\nto run this example, please install pytesseract and pdf2image - and there may be core libraries " + "that need to be installed as well - see comments above more details.") + + +def parsing_pdf_by_ocr (): + + print(f"Example - Parsing PDF with Scanned Pages") + + LLMWareConfig().set_active_db("sqlite") + + # Load the llmware sample files + print (f"\nstep 1 - loading the llmware sample files") + sample_files_path = Setup().load_sample_files() + print (f"step 2 - llmware sample files saved locally at: {sample_files_path}") + + # Parse individual documents. The output will be a list of blocks (dicts with metadata) + ingestion_file_path = os.path.join(sample_files_path,"Agreements") + + files = os.listdir(ingestion_file_path) + + parser = Parser() + + for i, doc in enumerate(files): + + t0 = time.time() + + print(f"\nProcessing file - {i} - {doc}") + + parser_output = parser.parse_one_pdf_by_ocr_images(ingestion_file_path, doc,save_history=True) + + if parser_output: + print(f"Completed parsing - {doc} - time - {time.time()-t0} - blocks created - {len(parser_output)}") + + # to see the full output of the blocks created - uncomment this section + """ + for j, entries in enumerate(parser_output): + print(f"parsed blocks created: {j} - {entries}") + """ + + return 0 + + +if __name__ == "__main__": + + x = parsing_pdf_by_ocr() + diff --git a/solutions/sources/parse_to_json.py b/solutions/sources/parse_to_json.py new file mode 100644 index 0000000..7593eda --- /dev/null +++ b/solutions/sources/parse_to_json.py @@ -0,0 +1,67 @@ + +"""This example demonstrates how to parse documents in memory and output directly to json file + -- Does not require the use of database + -- Useful for parsing-on-the-fly and then extracting output into another step in a pipeline +""" + +import os +from llmware.parsers import Parser +from llmware.setup import Setup + + +def parse_to_json(selected_folder): + + print(f"\nExample - Parsing Batch of Documents Directly to JSON file") + + # initialize default - will be used for output + my_json_file_name = "" + processed_files_list = [] + + # load the llmware sample files + print (f"\nstep 1 - loading the llmware sample files") + sample_files_path = Setup().load_sample_files() + input_folder = os.path.join(sample_files_path,selected_folder) + + # create a parser + parser = Parser() + + print (f"step 2 - parsing all of the files in parsing input folder: {input_folder}") + + # parse entire folder to json + parsing_output = parser.ingest_to_json(input_folder) + + # display the output + if parsing_output: + if "parser_output_filename" in parsing_output: + my_json_file_name = parsing_output["parser_output_filename"] + + if "processed_files" in parsing_output: + processed_files_list = parsing_output["processed_files"] + else: + print("update: unexpected error - parsing did not get completed") + + print (f"step 3 - parsing completed") + print (f" - parsing history file path: {parser.parser_folder}") + print (f" - parsing output created: {my_json_file_name}") + print (f" - files processed: {processed_files_list}") + print (f" - total blocks created: {len(parser.parser_output)}") # note: this is in the parser state + + return parsing_output + + +if __name__ == "__main__": + + # note on sample documents - downloaded by Setup() + # UN-Resolutions-500 is 500 pdf documents + # Invoices is 40 pdf invoice samples + # Agreements is ~15 contract documents + # AgreementsLarge is ~80 contract documents + # FinDocs is ~15 financial annual reports and earnings + # SmallLibrary is a mix of ~10 pdf and office documents + + # this is a list of document folders that will be pulled down by calling Setup() + sample_folders = ["Agreements", "Invoices", "UN-Resolutions-500", "SmallLibrary", "FinDocs", "AgreementsLarge"] + + selected_folder = sample_folders[0] + output = parse_to_json (selected_folder) + diff --git a/solutions/sources/parse_web_sources_in_memory.py b/solutions/sources/parse_web_sources_in_memory.py new file mode 100644 index 0000000..5c72f6b --- /dev/null +++ b/solutions/sources/parse_web_sources_in_memory.py @@ -0,0 +1,52 @@ + +""" This example demonstrates how to parse various web sources into a Library through HTML scraping. + + When parsing websites, please follow best practices, ethical guidelines and common sense - as a good example, + see https://monashdatafluency.github.io/python-web-scraping/section-5-legal-and-ethical-considerations/ + + To use the WebSite Parser requires several additional python libraries to be installed: + + pip3 install beautifulsoup4 + pip3 install lxml + pip3 install requests + pip3 install urllib3 + +""" + +from llmware.parsers import Parser, WebSiteParser + + +def parsing_web_sources_in_memory(): + + """ In this example. we will access the WebSiteParser through the general Parser class, with the main use + case of integrating a small HTML site into a library with inclusion of other file types. + + We recommend checking the website output first in memory, before automatically adding to a DB - as + usually the extracted text will require some post-processing to remove redundancies, potential formatting or JS - + and as a general safety check on the content. """ + + print(f"\nExample - Parsing Web Sources") + + # parse website directly - here are a few ideas for rapid testing + # please be respectful in keeping requests at low volume + + # high volume global website + website = "https://www.cnbc.com" + + # come visit NYC + website = "https://www.ny.com/general/centers.html" + # website = "https://bronxzoo.com" + # website = "https://en.wikipedia.org/wiki/Jalen_Brunson" + + website_parsed_output = Parser().parse_website(website, write_to_db=False, save_history=False, get_links=False) + + # look at the first 10 text blocks extracted + for x in range(0,min(10, len(website_parsed_output))): + print("text blocks extracted: ", website_parsed_output[x]) + + return 0 + + +if __name__ == "__main__": + + parsing_web_sources_in_memory() diff --git a/solutions/sources/parsing_microsoft_ir_docs.py b/solutions/sources/parsing_microsoft_ir_docs.py new file mode 100644 index 0000000..0385fce --- /dev/null +++ b/solutions/sources/parsing_microsoft_ir_docs.py @@ -0,0 +1,48 @@ + +""" This example shows how to parse .zip archives that include a mix of different file types - as an example, + we will use a set of Microsoft Investor Relations files, published on the Microsoft IR website - + + -- https://www.microsoft.com/en-us/investor + + We will pull a consolidated set of zip archives from llmware sample files repository that consists of + zip archives downloaded directly from the Microsoft IR set as the investor kit for each of the quarters + since 2020. + + Parsing ZIP archives is easy - they are automatically opened, and the source files are then routed to the + appropriate parser. + +""" + +from llmware.library import Library +from llmware.configs import LLMWareConfig +from llmware.setup import Setup +from llmware.retrieval import Query + +# feel free to use postgres or mongo, if installed +LLMWareConfig().set_active_db("sqlite") + +print("update: downloading and caching microsoft investor relations sample files") + +microsoft_ir = Setup().load_selected_sample_files(sample_folder="microsoft_ir") + +print("update: completed downloading files @ files path: ", microsoft_ir) + +my_lib = Library().create_new_library("microsoft_investor_relations_1") + +# pass the zip archives like any other file in .add_files method +parsing_output = my_lib.add_files(microsoft_ir) + +print("update: parsing output: ", parsing_output) + +# check out the images extracted +print("update: images extracted to path: ", my_lib.image_path) + +# optional - run an OCR against all of the images in the library - check out the example: +# -- examples/Parsing/ocr_embedded_doc_images.py + +# run a quick query +qr = Query(my_lib).text_query("azure", result_count=10) + +for i, res in enumerate(qr): + print("results: ", i, res) + diff --git a/solutions/sources/pdf_parser_new_configs.py b/solutions/sources/pdf_parser_new_configs.py new file mode 100644 index 0000000..c459fee --- /dev/null +++ b/solutions/sources/pdf_parser_new_configs.py @@ -0,0 +1,85 @@ + +""" Starting with llmware 0.2.7, new configuration options are exposed in the PDF parser, with many more to come. + This should not result in any breaking changes in existing code, but exposes new configuration options. + + Note: default encoding is now 'utf-8' which could result in changes from previous default 'ascii' -> if you + experience any issues, you can explicitly set the encoding = 'ascii' ... """ + +from llmware.library import Library + +fp = "/path/to/my/pdf/files" + +lib = Library().create_new_library("my_library") + +# standard call to 'ingest' files into a library (implicitly calls Parser and manages the details) +lib.add_files(input_folder_path=fp) + +# new configuration options in Parser class, exposed in .add_files method for convenience, although it can +# always be accessed by direct construction of a Parser + +# CHUNK SIZING - implemented in PDF parser (and will be supported fully in Office parser coming soon) +# Measured in string len characters, e.g., 400 characters as target text chunk size, with max of 600 characters + +# -- there are 3 chunking strategies supported: +# -- smart_chunking = 0 -> will stop at the target chunk size (or as close as possible) - breaks words +# -- smart_chunking = 1 -> will stop at the first white space after the target chunk size - preserves words +# -- smart_chunking = 2 -> will look for a natural break, either a "." or "\r" or "\n", up to the max size +# -- note: this is applied on a page-by-page basis, so if there are 1800 characters on a page, then in this example, +# it would result in 4 text chunks of ~400 characters, and a final chunk of the remainder of ~200 characters + +lib.add_files(input_folder_path=fp, chunk_size=400, max_chunk_size=600, smart_chunking=0) + +# CAPTURE PARAMETERS + +# -- ability to turn on/off the capture of other features in the parser, e.g, whether to capture images, tables, +# and formatted header text (e.g., BOLD, ITALICS and Large Font) + +# -- if you are only interested in text, then you can turn off these features for faster performance + +lib.add_files(input_folder_path=fp, get_images=False, get_tables=False, get_header_text=False) + +# ENCODING + +# -- encoding = "utf-8" - new default - supports Western European characters, and many characters across the +# wider Unicode set of code points - note: not full support yet for Asian languages and characters + +# -- backup option = "ascii" - if english only document, then this will often times result in a slightly cleaner +# output as it will restrict the output to ASCII 7-bit range (1-127) + +# -- backup option (not recommended, but available) = "latin-1" encoding - will attempt to save the 8-bit +# character directly into the database with UTF-8 bit mapping - which can create downstream validation problems +# but available if needed in a special case + +lib.add_files(input_folder_path=fp, encoding="utf-8") + +# STRIP HEADERS + +# -- some documents prepared by a 3rd party service will insert a repeating header at the top of each page +# -- if strip_header == True, then the parser will skip the first text box found on each page in an attempt to +# to remove repeating common header (very useful in some cases- and worth trying) + +lib.add_files(input_folder_path=fp, strip_header=True) + +# COPY FILES TO LIBRARY + +# -- by default, input files are collated to different parsers, and then at the end of the process, copied to a +# library file structure for convenient future access by the library and for downstream retrieval applications +# that query the library, and then want to provide direct access to the files. + +# -- if you do not want to duplicate files, then set copy_files_to_library = False + +lib.add_files(input_folder_path=fp, copy_files_to_library=False) + +# VERBOSE OUTPUT + +# -- four modes of output to screen +# -- verbose_level == 0 -> suppresses virtually all output, except major errors/problems +# -- verbose_level == 1 -> displays 1st ten pages of document text in text chunks as parsing +# -- verbose_level == 2 -> displays file name being parsed only +# -- verbose_level == 3 -> deep debugging mode (not recommended) - useful for llmware dev team in tracing errors + +lib.add_files(input_folder_path=fp, verbose_level=2) + + + + diff --git a/solutions/sources/pdf_table_extraction.py b/solutions/sources/pdf_table_extraction.py new file mode 100644 index 0000000..c9d276a --- /dev/null +++ b/solutions/sources/pdf_table_extraction.py @@ -0,0 +1,62 @@ + +""" This example illustrates how to ** extract financial tables from PDF documents ** + + extract_pdf_tables - shows end-to-end flow to automatically extract tables from PDFs + the sample documents (~15 financial documents - mostly 10Ks and annual reports) are available in public S3 repo + + this example is also reviewed in the llmware YouTube video 'Extract Tables from PDFs' + Check out this video on the llmware Youtube channel at: https://www.youtube.com/watch?v=YYcimVQEgO8&t=4s + +""" + +import os + +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def extract_pdf_tables(library_name): + + print(f"\nExample: Parsing Financial PDF Documents and Extracting Tables") + + # Step 1 - create library + print("\nstep 1 - create library - {}".format(library_name)) + + lib = Library().create_new_library(library_name) + + # Step 2 - pull down the sample files (or insert your own files here) + # --note: if you need to pull updated sample files, set 'over_write=True' + print("step 2 - pull sample files - FinDocs") + + sample_files_path = Setup().load_sample_files(over_write=True) + + # Step 3 - parse and extract all of the content from the Financial Documents + print("step 3 - parse, text chunk and text index the documents") + + parsing_output = lib.add_files(input_folder_path=os.path.join(sample_files_path, "FinDocs")) + + # review the parsing output summary info - all of the text and table blocks are in Mongo collection + print("update: parsing_output - ", parsing_output) + + # Step 4 - export all of the content into .jsonl files with metadata + output_fp = LLMWareConfig().get_tmp_path() + print("update: Step 4 - exporting all blocks into file path - ", output_fp) + + output1 = lib.export_library_to_jsonl_file(output_fp, "{}_export.jsonl".format(library_name)) + + # Step 5 - export all of the tables as csv with 'amazon' + print("update: Step 5 - exporting all tables with 'amazon' as csv files into file path - ", output_fp) + + output2 = Query(lib).export_all_tables(query="amazon", output_fp=output_fp) + + return output2 + + +if __name__ == "__main__": + + LLMWareConfig().set_active_db("sqlite") + + p = extract_pdf_tables("pdf_table_lib_example") + diff --git a/solutions/sources/query_state.py b/solutions/sources/query_state.py new file mode 100644 index 0000000..ae9a899 --- /dev/null +++ b/solutions/sources/query_state.py @@ -0,0 +1,63 @@ + +"""This example demonstrates the Query State mechanisms and ability to generate reports across multiple queries: + + 1. Create a library + 2. Run multiple queries - saving query state + 3. Generate reports from query state and export +""" + +import os +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def create_un_500_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "UN-Resolutions-500") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +# Demonstrate some methods involved with persisting and loading Query state as well as export +def query_state_and_export(library): + + # Create a Query instance with history persistence + query = Query(library, save_history=True) + + # Capture the query_id + query_id = query.query_id + + # Run a series of queries + query_results = query.text_query("sustainable development", result_count=20) + query_results = query.text_query("africa", result_count=26) + query_results = query.text_query("pandemic risk", result_count=15) + + # Save state - this will write the query state as JSON file in /query_history path + query.save_query_state() + + # Generate Retrieval Report. The report will be stored in the llmware_data/query_history folder + csv_file = query.generate_csv_report() + csv_file_path = os.path.join(LLMWareConfig().get_query_path(), csv_file) + + print (f"\nSaving Retrieval State and Export'") + print (f"Export for query id '{query_id}': {csv_file_path}") + + # Additionally here is how can clear state and reload based on a query_id: + query.clear_query_state() + query.load_query_state(query_id) + + return 0 + + +if __name__ == "__main__": + + lib = create_un_500_sample_library("lib_query_state_1") + query_state_and_export(lib) + + + diff --git a/solutions/sources/semantic_retrieval.py b/solutions/sources/semantic_retrieval.py new file mode 100644 index 0000000..ea61aaa --- /dev/null +++ b/solutions/sources/semantic_retrieval.py @@ -0,0 +1,85 @@ + +""" +This 'getting started' example demonstrates how to use basic semantic retrieval with the Query class + 1. Create a sample library + 2. Run a basic semantic query + 3. View the results +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def create_fin_docs_sample_library(library_name): + + print(f"update: creating library - {library_name}") + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "FinDocs") + parsing_output = library.add_files(ingestion_folder_path) + + print(f"update: building embeddings - may take a few minutes the first time") + + # note: if you have installed Milvus or another vector DB, please feel free to substitute + # note: if you have any memory constraints on laptop: + # (1) reduce embedding batch_size or ... + # (2) substitute "mini-lm-sbert" as embedding model + + library.install_new_embedding(embedding_model_name="industry-bert-sec", vector_db="chromadb",batch_size=200) + + return library + + +def basic_semantic_retrieval_example (library): + + # Create a Query instance + q = Query(library) + + # Set the keys that should be returned - optional - full set of keys will be returned by default + q.query_result_return_keys = ["distance","file_source", "page_num", "text"] + + # perform a simple query + my_query = "ESG initiatives" + query_results1 = q.semantic_query(my_query, result_count=20) + + # Iterate through query_results, which is a list of result dicts + print(f"\nQuery 1 - {my_query}") + for i, result in enumerate(query_results1): + print("results - ", i, result) + + # perform another query + my_query2 = "stock performance" + query_results2 = q.semantic_query(my_query2, result_count=10) + + print(f"\nQuery 2 - {my_query2}") + for i, result in enumerate(query_results2): + print("results - ", i, result) + + # perform another query + my_query3 = "cloud computing" + + # note: use of embedding_distance_threshold will cap results with distance < 1.0 + query_results3 = q.semantic_query(my_query3, result_count=50, embedding_distance_threshold=1.0) + + print(f"\nQuery 3 - {my_query3}") + for i, result in enumerate(query_results3): + print("result - ", i, result) + + return [query_results1, query_results2, query_results3] + + +if __name__ == "__main__": + + print(f"Example - Running a Basic Semantic Query") + + LLMWareConfig().set_active_db("sqlite") + + # step 1- will create library + embeddings with Financial Docs + lib = create_fin_docs_sample_library("lib_semantic_query_1") + + # step 2- run query against the library and embeddings + my_results = basic_semantic_retrieval_example(lib) diff --git a/solutions/sources/text_retrieval.py b/solutions/sources/text_retrieval.py new file mode 100644 index 0000000..1623c6f --- /dev/null +++ b/solutions/sources/text_retrieval.py @@ -0,0 +1,54 @@ + +""" +This 'getting started' example demonstrates how to use basic text retrieval with the Query class + 1. Create a sample library + 2. Run a basic text query + 3. View the results +""" + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def create_invoices_sample_library(library_name): + + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files(over_write=False) + ingestion_folder_path = os.path.join(sample_files_path, "Invoices") + parsing_output = library.add_files(ingestion_folder_path) + + return library + + +def basic_text_retrieval_example (library): + + # Step 2 - the Query class executes queries against a Library + + # Create a Query instance + q = Query(library) + + # Set the keys that should be returned - optional - full set of keys will be returned by default + q.query_result_return_keys = ["file_source", "page_num", "matches", "doc_ID", "block_ID", "content_type", "text"] + + # Perform a simple query + my_query = "total amount" + query_results = q.text_query(my_query, result_count=20) + + print(f"\nQuery: {my_query}") + + # Iterate through query_results, which is a list of result dicts + for i, result in enumerate(query_results): + print("results - ", i, result) + + return query_results + + +if __name__ == "__main__": + + print(f"\nExample - Basic Text Query") + + lib = create_invoices_sample_library("lib_text_retrieval_example_2") + + my_results = basic_text_retrieval_example(lib) diff --git a/solutions/ui/SLIM_extract_tool_streamlit.py b/solutions/ui/SLIM_extract_tool_streamlit.py new file mode 100644 index 0000000..cbe39a7 --- /dev/null +++ b/solutions/ui/SLIM_extract_tool_streamlit.py @@ -0,0 +1,56 @@ +""" This example provides a basic framework to build a Chatbot UI interface in conjunction with LLMWare + using Streamlit Chat UI. + + To run this example requires an install of Streamlit, e.g., `pip3 install streamlit` + + To execute the script, run from the command line with: `streamlit run using_with_streamlit_ui.py` + + Also, please note that the first time you run with a new model, the model will be downloaded and cached locally, + so expect a delay on the 'first run' which will be much faster on every successive run. + + All components of the chatbot will be running locally, so the speed will be determined greatly by the + CPU/GPU capacities of your machine. + + We have set the max_output at 250 tokens - for faster, set lower ... + + For more information on the Streamlit Chat UI, + see https://docs.streamlit.io/develop/tutorials/llms/build-conversational-apps + + +""" + +import streamlit as st +from llmware.models import ModelCatalog + +# Title of the Streamlit app +st.title("SLIM Extract Tool LLMWARE") + +# Text input for the text to analyze +text_to_analyze = st.text_area("Enter the text to analyze:", "\"Good Will Hunting,\" a 1997 film directed by Gus Van Sant, tells the story of a young janitor at MIT who has a hidden talent for mathematics and undergoes therapy to confront his troubled past.") + +# Text input for the queries +queries_input = st.text_area("Enter your queries (comma separated):", "Director, Film Name") + +# Convert the input queries to a list +queries_list = [query.strip() for query in queries_input.split(',')] + +# Button to run the analysis +if st.button("Analyze"): + # Load the model + model = ModelCatalog().load_model("slim-extract-tool", sample=False, temperature=0.0, max_output=250) + + # Initialize the output dictionary + output_dict = {} + + # Loop through the queries and call the model with the entire text for each query + for j, query in enumerate(queries_list): + st.write(f"Query {j+1}: {query}") + response = model.function_call(text_to_analyze, function="extract", params=[query]) + output_dict.update(response["llm_response"]) + #if not response["llm_response"]: + # st.write("No response") + # st.write("Extract response: ", response["llm_response"]) + + # Display the response on the screen + st.write("Output Dictionary:") + st.json(output_dict) \ No newline at end of file diff --git a/solutions/ui/dueling_chatbot.py b/solutions/ui/dueling_chatbot.py new file mode 100644 index 0000000..e13052a --- /dev/null +++ b/solutions/ui/dueling_chatbot.py @@ -0,0 +1,316 @@ + +""" This example shows how to build a unique 'Dueling Q&A ChatBot' in which a question-generating model 'chats' +with a question-answering model using a selected context passage from the user. + + The user provides input of selecting the context passage, and then typing "Go" on the prompt bar, and the +'dueling' bots take it from there ... + + Please note that two models will be downloaded and cached locally on first use - so please expect 1-2 minutes +for the first run, and then much faster loads on subsequent tries. + + This example uses Streamlit for the UI. If you are new to using Steamlit, to run this example: + + 1. `pip3 install streamlit` + + 2. to run, go to the command line: streamlit run "path/to/dueling_chatbot.py" + +""" + +import streamlit as st +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs + +GGUFConfigs().set_config("max_output_tokens", 500) + +if "question_history" not in st.session_state: + st.session_state["question_history"] = [] + + +# test passage pulled from CNBC news story on Tuesday, May 28, 2024 +test_passage = ("OpenAI said Tuesday it has established a new committee to make recommendations to the " + "company’s board about safety and security, weeks after dissolving a team focused on AI safety. " + "In a blog post, OpenAI said the new committee would be led by CEO Sam Altman as well as " + "Bret Taylor, the company’s board chair, and board member Nicole Seligman. The announcement " + "follows the high-profile exit this month of an OpenAI executive focused on safety, " + "Jan Leike. Leike resigned from OpenAI leveling criticisms that the company had " + "under-invested in AI safety work and that tensions with OpenAI’s leadership had " + "reached a breaking point.") + + +@st.cache_resource +def load_question_model(temperature=0.5): + """ Loads the Question Generating Model. """ + question_model = ModelCatalog().load_model("slim-q-gen-tiny-tool", + temperature=temperature, + sample=True) + return question_model + + +@st.cache_resource +def load_answer_model(): + """ Loads the Answering Model. """ + answer_model = ModelCatalog().load_model("bling-phi-3-gguf",temperature=0.0, sample=False) + return answer_model + + +def get_new_question(q_model, question_type, test_passage): + + new_q = "" + max_tries = 10 + tries = 0 + + while not new_q or new_q in st.session_state["question_history"]: + + response = q_model.function_call(test_passage, params=[question_type], get_logits=False) + if response: + if "llm_response" in response: + if "question" in response["llm_response"]: + new_q = response["llm_response"]["question"] + if isinstance(new_q, list) and len(new_q) > 0: + new_q = new_q[0] + if new_q not in st.session_state["question_history"]: + st.session_state["question_history"].append(new_q) + break + + tries += 1 + if tries >= max_tries: + break + + return new_q + + +def get_new_answer(question, ans_model, test_passage): + + response = ans_model.inference(question, add_context=test_passage) + answer = response["llm_response"] + + return answer + + +def get_input_passage(sample_passage_name, custom_passage_text): + + if sample_passage_name != "None": + + if sample_passage_name == "OpenAI": + + return ("OpenAI said Tuesday it has established a new committee to make recommendations to the " + "company’s board about safety and security, weeks after dissolving a team focused on AI safety. " + "In a blog post, OpenAI said the new committee would be led by CEO Sam Altman as well as " + "Bret Taylor, the company’s board chair, and board member Nicole Seligman. The announcement " + "follows the high-profile exit this month of an OpenAI executive focused on safety, " + "Jan Leike. Leike resigned from OpenAI leveling criticisms that the company had " + "under-invested in AI safety work and that tensions with OpenAI’s leadership had " + "reached a breaking point. The name of the new committee is the AI Safety Committee.") + + elif sample_passage_name == "Apple": + + return ("Apple shares popped 5% to a new record high of around $203 per share on Tuesday, a day " + "after the company announced its long-awaited push into artificial intelligence at its annual " + "developer conference on Monday. Apple introduced a range of new AI features during the event, " + "including an overhaul of its voice assistant Siri, integration with OpenAI’s ChatGPT, " + "a range of writing assistance tools and new customizable emojis. The company pitched the " + "features as AI for the average person, though users will likely need to upgrade their " + "iPhones to access the tools. With Tuesday’s share move, Apple bested its previous record " + "from Dec. 14. The company’s developer conference came as a welcome sign for investors who " + "have been watching to see how Apple will capitalize on the ongoing AI boom. Analysts from " + "Morgan Stanley said Apple’s AI features strongly position the company with “the most " + "differentiated consumer digital agent.” Additionally, the analysts believe that the " + "features will drive consumers to upgrade their iPhones, which should “accelerate " + "device replacement cycles.” They said Apple will still have to deliver when the AI " + "features are first available in the fall, but they think the “building blocks are in " + "place for a return to growth and more sustained outperformance.") + + elif sample_passage_name == "Los Angeles Lakers": + + return ("The Lakers have finished better than seventh in the Western Conference standings just once " + "in the past 12 seasons (when they won the title in 2019-20). Their franchise player, " + "LeBron James, turns 40 in December. They have no salary-cap space unless James were " + "to leave in free agency (he has until June 29 to make a decision on his $51.4 million " + "player option). They have limited trade assets. They play in a super-competitive conference " + "where the teams behind them are upwardly mobile and aggressive and most of the teams in " + "front of them are going to continue to be good -- or get even better -- in the " + "immediate future. And they have a massive and highly demanding fanbase and are " + "under a constant microscope by the national media because they drive massive " + "audience engagement across the world. Vogel won a title in 2020. Darvin Ham reached the " + "Western Conference finals in 2023. Neither lasted longer than three seasons. No " + "Lakers coach has since Phil Jackson has lasted more than three seasons. These are some of " + "the factors Hurley undoubtedly was weighing before making his choice over the weekend. " + "It's hard to even quantify what would be considered a successful season for the Lakers " + "in 2024-25 without knowing what changes are made to the roster. Avoiding the play-in tournament, " + "frankly, would be a reasonable if challenging goal.") + + elif sample_passage_name == "Buy Home": + + return ("The price for owning a home is rising rapidly and not just the mortgage payments. " + "US homeowners are now paying an average of $18,118 a year on property taxes, homeowners’ " + "insurance, maintenance, energy and various other expenses linked to owning a home, " + "according to a new Bankrate study. That’s nearly the cost to buy a used car and represents " + "a 26% increase from four years ago when it cost $14,428 annually to own and maintain a home. " + "All of these variable expenses are on top of the fixed cost of a mortgage, including " + "property taxes, homeowners insurance, energy costs, internet, cable bills and " + "home maintenance. The findings are another reminder of how much more expensive life " + "has become since Covid-19. Many Americans would like to buy a home but have been unable to " + "because home prices have spiked to record highs and mortgage rates remain elevated. " + "The housing market is historically unaffordable. But even the ones fortunate enough to " + "have bought a home over the past few years are grappling with sticker shock over the cost " + "of maintaining it. The per-month cost of owning and maintaining a home has gone from " + "$1,202 a month in 2020 to $1,510 now, Bankrate found.") + + elif sample_passage_name == "Vacation": + + return ("Temperatures are rising. Hotel prices are exploding. And travelers are already behaving badly. " + "Welcome to another summer in Europe. From the headlines, things already look chaotic. " + "Famous sites are raising their entry fees. Hotel rooms are like gold dust. And the " + "dollar has slipped against both the pound and the euro. Oh, and there’s the small matter " + "of crowds. “There’s been a substantial increase on last year’s demand,” says Tom " + "Jenkins, CEO of the European Tourism Organisation, speaking about US travelers to " + "Europe. “2023 saw higher numbers than 2019, and this year we’re comfortably seeing more – " + "record volumes of Americans coming to Europe.” Kayla Zeigler agrees. As the owner of " + "Destination Europe, she is sending “record numbers” of clients to the continent this year. " + "Graham Carter, director of Unforgettable Travel, a tour operator with a 90% US " + "client base, says that many guests are finding the idea of Europe prohibitively " + "expensive this year. People are wondering, is Europe worth it?” he says. “It’s " + "booking up in advance and prices are quite high. There’s been such a huge demand " + "for travel in the past three years, and lots of places are pushing up prices.” " + "Is summer in Europe already a washout? According to the experts, that all depends " + "on what kind of sacrifices you’re prepared to make. A weak dollar First things first: " + "travelers from the US are already at a disadvantage due to a weak dollar. Against " + "the euro, $1 was worth around 91 or 92 euro cents as of June 5, at mid-market rates. " + "Sure, that’s better than the December 2020-January 2021 five-year low when the " + "dollar was hovering around 82 cents. But it’s also down from a year earlier, " + "when a dollar was worth about 95 euro cents – and it’s way down from last " + "September’s five-year high when it peaked at 1.04 euros, according to currency " + "conversion specialists Wise. For those traveling to the UK it’s a similar state of " + "affairs. This time last year, $1 netted travelers 80 pence. As of Wednesday, it was " + "78p – a fall from the September peak of nearly 83p. The dollar is also down, year on year, " + "against 11 more European currencies. From Bosnia to Bulgaria, Denmark to Iceland, " + "Poland to Romania and Sweden to Switzerland, travelers changing dollars will be worse off. " + "While a few cents to the dollar doesn’t sound much on a single transaction, the small " + "drops can make a difference on credit card bills on the return home. A 500 euro hotel " + "room equates to $543 at Friday’s mid-market exchange rate, where it would have been " + "$480 in September.") + + elif sample_passage_name == "Taylor Swift": + + return ("Taylor Swift stopped her concert in Edinburgh, Scotland, on Friday to help a fan. Swift was " + "in the middle of singing her “Midnights” song “Would’ve Could’ve Should’ve” when she noticed " + "a fan who was in distress. In a video that went viral on social media, the singer-songwriter " + "is seen requesting assistance for the fan. “We need help right in front of me, please, " + "right in front of me,” Swift sang while playing her guitar and keeping her eyes locked on " + "the fan. “Just gonna keep playing until we notice where it is. Swift continued strumming " + "her guitar while motioning over to the person in need of help.") + + else: + return custom_passage_text + else: + return custom_passage_text + + +def ask_and_answer_game(source_passage, q_model="slim-q-gen-tiny-tool", number_of_tries=10, question_type="question", + temperature=0.5): + + """ Shows a simple two model game of using q-gen model to generate a question, and then a second model + to answer the question generated. """ + + # this is the model that will generate the 'question' + q_model = ModelCatalog().load_model(q_model, sample=True, temperature=temperature) + + # this will be the model used to 'answer' the question + answer_model = ModelCatalog().load_model("bling-phi-3-gguf") + + questions = [] + + print(f"\nGenerating a set of questions automatically from the source passage.\n") + + for x in range(0,number_of_tries): + + response = q_model.function_call(source_passage, params=[question_type], get_logits=False) + + if response: + if "llm_response" in response: + if "question" in response["llm_response"]: + new_q = response["llm_response"]["question"] + + # only keep new questions + if new_q and new_q not in questions: + questions.append(new_q) + + print(f"inference - {x} - response: {response}") + + print("\nAnswering the generated questions\n") + for i, question in enumerate(questions): + + print(f"\nquestion: {i} - {question}") + if isinstance(question, list) and len(question) > 0: + response = answer_model.inference(question[0], add_context=test_passage) + print(f"response: ", response["llm_response"]) + + return True + + +def ask_and_answer_dueling_bots_ui_app (input_passage): + + question_model = "slim-q-gen-phi-3-tool" + answer_model = "bling-stablelm-3b-tool" + + st.title(f"Ask and Answer Dueling Bots") + st.write(f"Asking the questions: {question_model}") + st.write(f"Answering the questions: {answer_model}") + + question_model = load_question_model() + answer_model = load_answer_model() + + with st.sidebar: + + st.write("Today's Subject") + sample_passage = st.selectbox("sample_passage", ["OpenAI", "Apple", "Los Angeles Lakers", "Buy Home", + "Taylor Swift", "Vacation", "None"], index=0) + # custom_passage = st.text_area(label="Subject",value=input_passage,height=100) + custom_passage = test_passage + + input_passage = get_input_passage(sample_passage,custom_passage) + + mode = st.selectbox("mode", ["question", "boolean", "multiple choice"],index=0) + + number_of_tries = st.selectbox("tries", [5,10,20],index=0) + + st.write(input_passage) + + # initialize chat history + if "messages" not in st.session_state: + st.session_state.messages = [] + + # display chat messages from history on app rerun + for message in st.session_state.messages: + with st.chat_message(message["role"]): + st.markdown(message["content"]) + + # accept user input + prompt = st.chat_input("Say 'Go' and the Bots will Start") + if prompt: + + for x in range(0, number_of_tries): + + with st.chat_message("user"): + + new_question = get_new_question(question_model,mode, input_passage) + st.markdown(new_question) + + with st.chat_message("assistant"): + + new_answer = get_new_answer(new_question,answer_model, input_passage) + st.markdown(new_answer) + + st.session_state.messages.append({"role": "user", "content": new_question}) + st.session_state.messages.append({"role": "assistant", "content": new_answer}) + + return 0 + + +if __name__ == "__main__": + + ask_and_answer_dueling_bots_ui_app(test_passage) + + + + diff --git a/solutions/ui/gguf_streaming_chatbot.py b/solutions/ui/gguf_streaming_chatbot.py new file mode 100644 index 0000000..70c7618 --- /dev/null +++ b/solutions/ui/gguf_streaming_chatbot.py @@ -0,0 +1,76 @@ + +""" This example shows how to build a local chatbot prototype using llmware and Streamlit. The example shows +how to use several GGUF chat models in the LLMWare catalog, along with using the model.stream method which +provides a real time generator for displaying the bot response in real-time. + + This is purposefully super simple script (but surprisingly fun) to provide the core of the recipe. + + The Streamlit code below is derived from Streamlit tutorials available at: + https://docs.streamlit.io/develop/tutorials/llms/build-conversational-apps + + If you are new to using Steamlit, to run this example: + + 1. pip3 install streamlit + + 2. to run, go to the command line: streamlit run "path/to/gguf_streaming_chatbot.py" + +""" + +import streamlit as st +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs + +GGUFConfigs().set_config("max_output_tokens", 500) + + +def simple_chat_ui_app (model_name): + + st.title(f"Simple Chat with {model_name}") + + model = ModelCatalog().load_model(model_name, temperature=0.3, sample=True, max_output=450) + + # initialize chat history + if "messages" not in st.session_state: + st.session_state.messages = [] + + # display chat messages from history on app rerun + for message in st.session_state.messages: + with st.chat_message(message["role"]): + st.markdown(message["content"]) + + # accept user input + prompt = st.chat_input("Say something") + if prompt: + + with st.chat_message("user"): + st.markdown(prompt) + + with st.chat_message("assistant"): + + # note that the st.write_stream method consumes a generator - so pass model.stream(prompt) directly + bot_response = st.write_stream(model.stream(prompt)) + + st.session_state.messages.append({"role": "user", "content": prompt}) + st.session_state.messages.append({"role": "assistant", "content": bot_response}) + + return 0 + + +if __name__ == "__main__": + + # a few representative good chat models that can run locally + # note: will take a minute for the first time it is downloaded and cached locally + + chat_models = ["phi-3-gguf", + "llama-2-7b-chat-gguf", + "llama-3-instruct-bartowski-gguf", + "openhermes-mistral-7b-gguf", + "zephyr-7b-gguf", + "tiny-llama-chat-gguf"] + + model_name = chat_models[0] + + simple_chat_ui_app(model_name) + + + diff --git a/solutions/ui/integrating_pywebio_ui.py b/solutions/ui/integrating_pywebio_ui.py new file mode 100644 index 0000000..06694a4 --- /dev/null +++ b/solutions/ui/integrating_pywebio_ui.py @@ -0,0 +1,482 @@ + +""" This example is a fast start to show the power of leveraging PyWebio for rapid easy-to-integrate UIs ... + + The text below is a standard example that runs a set of inferences and prints output to the console. + + By adding two lines at the top of the script, we now have a simple web UI. + + To run this example: + + `pip3 install pywebio` + + Check out pywebio docs for lots more examples of easily integrating more customized graphical elements. + + https://pywebio.readthedocs.io/en/latest/ + + Look for more examples coming soon + welcome creative ideas and contributions from the community! + + """ + +import time +from llmware.prompts import Prompt + +from pywebio.output import put_text + +# one line change - assigns standard python 'print' to 'put_text' +# experiment by commenting this out +print = put_text + + +def hello_world_questions(): + + """ Representative test set - we would recommend running this script as a 'hello world' test on the first + try that you use a model, and then adapt the content to your own set of context and questions. + + There is nothing special about these questions, and in fact, you will note that many of the models will get + a couple of answers wrong (especially the ~1B parameter models). The errors are important insights + as you evaluate which models to consider for your use case. + + To adapt this test set, just create your own list with dictionary entries and keys 'query', 'answer' and 'context'. + + """ + + test_list = [ + + {"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"}, + + {"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"} + + ] + + return test_list + + +def llmware_bling_dragon_hello_world (model_name): + + """ Simple inference loop that loads a model and runs through a series of test questions. """ + + t0 = time.time() + test_list = hello_world_questions() + + print(f"\n > Loading Model: {model_name}...") + + # please note that by default, we recommend setting temperature=0.0 and sample=False for fact-based RAG + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1 -t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{ i +1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"] ,context=entries["context"], prompt_name="default_with_context") + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + print(f"Gold Answer: {entries['answer']}") + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2 -t1} seconds") + + return 0 + + +if __name__ == "__main__": + + bling_pytorch = [ + + # pytorch models - will run fast on GPU, and smaller ones good for CPU only + # note: you will need to install pytorch and transformers to pull and access these models + + "llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/bling-1.4b-0.1", + "llmware/bling-falcon-1b-0.1", + "llmware/bling-cerebras-1.3b-0.1", + "llmware/bling-sheared-llama-1.3b-0.1", + "llmware/bling-sheared-llama-2.7b-0.1", + "llmware/bling-red-pajamas-3b-0.1", + "llmware/bling-stable-lm-3b-4e1t-v0", + "llmware/bling-phi-3", + "llmware/bling-phi-3.5" + ] + + dragon_pytorch = [ + + # pytorch models - intended for GPU server use - will require pytorch, transformers, and in some cases, + # other dependencies (einops, flash_attn). + + "llmware/dragon-mistral-7b-v0", + "llmware/dragon-yi-6b-v0", + "llmware/dragon-qwen-7b", + "llmware/dragon-llama-7b-v0", + "llmware/dragon-mistral-0.3", + "llmware/dragon-llama-3.1", + "llmware/dragon-deci-7b-v0" + ] + + bling_gguf = [ + + # smaller cpu-oriented models - optimal for running on a CPU + + "bling-phi-3.5-gguf", # **NEW** - phi-3.5 (3.8b) + "bling-answer-tool", # this is quantized bling-tiny-llama (1.1b) + "bling-qwen-0.5b-gguf", # **NEW** - qwen2 (0.5b) + "bling-qwen-1.5b-gguf", # **NEW** - qwen2 (1.5b) + "bling-stablelm-3b-tool", # quantized bling-stablelm-3b (2.7b) + "bling-phi-3-gguf", # quantized phi-3 (3.8b) + "bling-phi-2-gguf", # quantized phi-2 (2.7b) + ] + + dragon_gguf = [ + + # larger models - 6b - 9b + + "dragon-yi-answer-tool", # quantized yi-6b (v1) (6b) + "dragon-llama-answer-tool", + "dragon-mistral-answer-tool", + "dragon-qwen-7b-gguf", # **NEW** qwen2-7b (7b) + "dragon-yi-9b-gguf", # **NEW** yi-9b (8.8b) + "dragon-llama-3.1-gguf", + "dragon-mistral-0.3-gguf" + + ] + + llmware_bling_dragon_hello_world("bling-answer-tool") + + diff --git a/solutions/ui/multimedia_bot.py b/solutions/ui/multimedia_bot.py new file mode 100644 index 0000000..424243e --- /dev/null +++ b/solutions/ui/multimedia_bot.py @@ -0,0 +1,169 @@ + +""" This example shows a multimedia bot created in less than 100 lines of code that +leverages the CPU, GPU and NPU + + -- designed to run on an AI PC with Intel Lunar Lake with CPU, GPU and NPU + -- if you do not have GPU, it will auto-fallback to CPU + -- if you do not have NPU, you can change the option to GPU + + To run this example, we will need the following dependencies in addition to llmware: + + -- pip3 install openvino_genai + -- pip3 install pywebio + +""" + +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + +import os +import threading + +from pywebio.input import input_group, textarea, actions +from pywebio.output import put_text, put_markdown, put_image, use_scope, put_info +from pywebio.session import set_env + + +def text_gen_bot(**kwargs): + + """ Simple text generation streaming bot - will run using GGUF on CPU """ + + user_msg = kwargs.get("user_msg", "") + img_counter = kwargs.get("img_counter", 0) + + # llmware load_model + text_gen_model = ModelCatalog().load_model("phi-3-gguf", + max_output=200) + + inst = "Complete this story: " + prompt = inst + user_msg + text_output = "" + + with use_scope(f"text_gen" + str(img_counter)): + + # llmware stream generation + for token in text_gen_model.stream(prompt): + put_text(token, inline=True) + text_output += token + + put_text("\nTo be continued ...") + + # for demo example, we will write the text from the thread to a tmp file + fp = os.path.join(LLMWareConfig().get_llmware_path(), "txt_tmp.txt") + if os.path.exists(fp): + os.remove(fp) + f = open(fp, "w") + f.write(text_output) + f.close() + + return text_output + + +def image_gen_bot(**kwargs): + + """ Image generation bot that will run on GPU. """ + + user_msg = kwargs.get("user_msg", "") + img_counter = kwargs.get("img_counter", 0) + + # llmware load_model + model = ModelCatalog().load_model("lcm-dreamshaper-ov") + + inst = "Draw an image: " + prompt = inst + user_msg + + # specialized pipeline on the model + img_path = model.text_to_image_gen(prompt, f"test_image_{img_counter}") + content = open(img_path, "rb").read() + + # display the image on the screen with pywebio + with use_scope(f"img_gen" + str(img_counter)): + put_image(content) + + return img_path + + +def classifier_agent_bot(**kwargs): + + """ Simple classification agent running on NPU """ + + text_output = kwargs.get("text_output", "") + npu_model = kwargs.get("npu_model", None) + + # pass the model to the thread - and execute a function call + response = npu_model.function_call(text_output) + + put_text("\n\nNPU Classification Agent: " + str(response["llm_response"])) + + return True + + +def run_bot(): + + """ Main function - starts a user prompt loop, and then kicks off + three threads in parallel on CPU, GPU and NPU. """ + + set_env(input_panel_fixed=False, output_animation=False) + put_markdown("""# Multimedia Bot with LLMWare, OpenVINO, & PyWebio""") + + img_counter = 0 + start_bot = True + + while start_bot: + + # user input chat box + + form = input_group('', [ + textarea(name='msg', placeholder='Ask LLMWare Bot', rows=3), + actions(name='cmd', buttons=['Send', 'Exit']) + ]) + + if form['cmd'] == "Exit": + start_bot = False + break + + user_msg = form['msg'] + + # display the user prompt + put_info(user_msg) + + # thread 1 - CPU - text gen + text_gen_thread = threading.Thread(target=text_gen_bot, + kwargs={"user_msg": user_msg, + "img_counter": img_counter}) + text_gen_thread.start() + + # thread 2 - GPU - text to image gen + image_gen_thread = threading.Thread(target=image_gen_bot, + kwargs={"user_msg": user_msg, + "img_counter": img_counter}) + image_gen_thread.start() + + # load the npu model in main and pass to thread + npu_model = ModelCatalog().load_model("slim-topics-npu-ov", + sample=False,temperature=0.0, + device="NPU") + + image_gen_thread.join() + text_gen_thread.join() + + # pull the text output file created in the text gen thread + fp = os.path.join(LLMWareConfig().get_llmware_path(), "txt_tmp.txt") + text_output = "" + if os.path.exists(fp): + text_output = open(fp, "r").read() + + # kick off NPU thread + npu_gen_thread = threading.Thread(target=classifier_agent_bot, + kwargs={"text_output": text_output, + "npu_model": npu_model}) + + npu_gen_thread.start() + + img_counter += 1 + + return True + + +if __name__ == "__main__": + run_bot() diff --git a/solutions/ui/rag_ui_with_query_topic_with_streamlit.py b/solutions/ui/rag_ui_with_query_topic_with_streamlit.py new file mode 100644 index 0000000..4b7ca7a --- /dev/null +++ b/solutions/ui/rag_ui_with_query_topic_with_streamlit.py @@ -0,0 +1,94 @@ + +""" This example shows how to build a simple UI RAG application for longer documents in which a retrieval query step + is required to build a context from selected text chunks in the document. + + This example is build with a Streamlit UI. To run, it requires a separate `pip install streamlit`, and + to execute the script, you should run from the command line with: + + `streamlit run using_with_streamlit_ui.py` + + For more information about Streamlit, check out their docs: https://docs.streamlit.io/develop/tutorials + + To build out the application, you would replace the very simple 'text search' mechanism used below with + techniques outlined in examples in Embeddings and Retrieval. + +""" + + +import os +import streamlit as st + +from llmware.prompts import Prompt +from llmware.setup import Setup + +# st.set_page_config(layout="wide") + + +def simple_analyzer_with_topic_query (): + + st.title("Simple RAG Analyzer with Focusing Query") + + prompter = Prompt() + + sample_files_path = Setup().load_sample_files(over_write=False) + doc_path = os.path.join(sample_files_path, "Agreements") + + files = os.listdir(doc_path) + file_name = st.selectbox("Choose an Agreement", files) + + # ** topic_query ** = this is a proxy for a more complex focusing retrieval strategy to target only a + # specific part of the document, rather then the whole document + # in this case, this will run a 'text match' search against the topic query to reduce the + # text chunks reviewed in trying to answer the question + + topic_query = st.text_area("Filtering Topic (hint: 'vacation')") + + # ** prompt_text ** - this is the question that will be passed to the LLM + prompt_text = st.text_area("Question (hint: 'how many vacation days will the executive receive'") + + model_name = st.selectbox("Choose a model for answering questions", ["bling-phi-3-gguf", + "bling-tiny-llama-1b", + "bling-stablelm-3b-tool", + "llama-3-instruct-bartowski-gguf", + "dragon-llama-answer-tool"]) + + if st.button("Run Analysis"): + + if file_name and prompt_text and model_name: + + prompter.load_model(model_name, temperature=0.0, sample=False) + + # parse the PDF in memory and attach to the prompt + if not topic_query: + sources = prompter.add_source_document(doc_path,file_name) + else: + # this is where we use the topic_query to filter the parsed document + sources = prompter.add_source_document(doc_path,file_name, query=topic_query) + + # run the inference with the source + response = prompter.prompt_with_source(prompt_text) + + # fact checks + fc = prompter.evidence_check_numbers(response) + cs = prompter.evidence_check_sources(response) + + if len(response) > 0: + if "llm_response" in response[0]: + response = response[0]["llm_response"] + + st.write(f"Answer: {response}") + + if len(fc) > 0: + if "fact_check" in fc[0]: + fc_out = fc[0]["fact_check"] + st.write(f"Numbers Check: {fc_out}") + + if len(cs) > 0: + if "source_review" in cs[0]: + sr_out = cs[0]["source_review"] + st.write(f"Source review: {sr_out}") + + +if __name__ == "__main__": + + simple_analyzer_with_topic_query() diff --git a/solutions/ui/simple_rag_ui_with_streamlit.py b/solutions/ui/simple_rag_ui_with_streamlit.py new file mode 100644 index 0000000..b0f3185 --- /dev/null +++ b/solutions/ui/simple_rag_ui_with_streamlit.py @@ -0,0 +1,86 @@ + +""" This example shows how to build a simple RAG application with UI with Streamlit and LLMWare. + + Note: it requires a separate `pip install streamlit`, and to run the script, you should run from the + command line with: + + `streamlit run using_with_streamlit_ui.py` + + For this example, we will be prompting against a set of Invoice documents, provided in the LLMWare + sample files. + + If you would like to substitute longer documents then please look at the UI example: + -- rag_ui_with_query_topic_with_streamlit.py + + as a framework to get started integrating a retrieval step before the prompt of the source + + For more information about Streamlit, check out their docs: https://docs.streamlit.io/develop/tutorials + +""" + + +import os +import streamlit as st + +from llmware.prompts import Prompt +from llmware.setup import Setup + +# st.set_page_config(layout="wide") + + +def simple_analyzer (): + + st.title("Simple RAG Analyzer") + + prompter = Prompt() + + sample_files_path = Setup().load_sample_files(over_write=False) + doc_path = os.path.join(sample_files_path, "Invoices") + + files = os.listdir(doc_path) + file_name = st.selectbox("Choose an Invoice", files) + + prompt_text = st.text_area("Question (hint: 'what is the total amount of the invoice?'") + + model_name = st.selectbox("Choose a model for answering questions", ["bling-phi-3-gguf", + "bling-tiny-llama-1b", + "bling-stablelm-3b-tool", + "llama-3-instruct-bartowski-gguf", + "dragon-llama-answer-tool"]) + + if st.button("Run Analysis"): + + if file_name and prompt_text and model_name: + + prompter.load_model(model_name, temperature=0.0, sample=False) + + # parse the PDF in memory and attach to the prompt + sources = prompter.add_source_document(doc_path,file_name) + + # run the inference with the source + response = prompter.prompt_with_source(prompt_text) + + # fact checks + fc = prompter.evidence_check_numbers(response) + cs = prompter.evidence_check_sources(response) + + if len(response) > 0: + if "llm_response" in response[0]: + response = response[0]["llm_response"] + + st.write(f"Answer: {response}") + + if len(fc) > 0: + if "fact_check" in fc[0]: + fc_out = fc[0]["fact_check"] + st.write(f"Numbers Check: {fc_out}") + + if len(cs) > 0: + if "source_review" in cs[0]: + sr_out = cs[0]["source_review"] + st.write(f"Source review: {sr_out}") + + +if __name__ == "__main__": + + simple_analyzer() diff --git a/solutions/ui/using_streamlit_chat_ui.py b/solutions/ui/using_streamlit_chat_ui.py new file mode 100644 index 0000000..a3f7010 --- /dev/null +++ b/solutions/ui/using_streamlit_chat_ui.py @@ -0,0 +1,81 @@ + +""" This example provides a basic framework to build a Chatbot UI interface in conjunction with LLMWare + using Streamlit Chat UI. + + To run this example requires an install of Streamlit, e.g., `pip3 install streamlit` + + To execute the script, run from the command line with: `streamlit run using_with_streamlit_ui.py` + + Also, please note that the first time you run with a new model, the model will be downloaded and cached locally, + so expect a delay on the 'first run' which will be much faster on every successive run. + + All components of the chatbot will be running locally, so the speed will be determined greatly by the + CPU/GPU capacities of your machine. + + We have set the max_output at 250 tokens - for faster, set lower ... + + For more information on the Streamlit Chat UI, + see https://docs.streamlit.io/develop/tutorials/llms/build-conversational-apps + + +""" + + +import streamlit as st +from llmware.models import ModelCatalog + + +def simple_chat_ui_app (model_name): + + st.title(f"Simple Chat with {model_name}") + + model = ModelCatalog().load_model(model_name, temperature=0.3, sample=True, max_output=250) + + # initialize chat history + if "messages" not in st.session_state: + st.session_state.messages = [] + + # display chat messages from history on app rerun + for message in st.session_state.messages: + with st.chat_message(message["role"]): + st.markdown(message["content"]) + + # accept user input + prompt = st.chat_input("Say something") + if prompt: + + with st.chat_message("user"): + st.markdown(prompt) + + with st.chat_message("assistant"): + + model_response = model.inference(prompt) + + # insert additional error checking / post-processing of output here + bot_response = model_response["llm_response"] + + st.markdown(bot_response) + + st.session_state.messages.append({"role": "user", "content": prompt}) + st.session_state.messages.append({"role": "assistant", "content": bot_response}) + + return 0 + + +if __name__ == "__main__": + + # a few representative good chat models that can run locally + # note: will take a minute for the first time it is downloaded and cached locally + + chat_models = ["phi-3-gguf", + "llama-2-7b-chat-gguf", + "llama-3-instruct-bartowski-gguf", + "openhermes-mistral-7b-gguf", + "zephyr-7b-gguf"] + + model_name = chat_models[0] + + simple_chat_ui_app(model_name) + + + diff --git a/solutions/use_cases/README.md b/solutions/use_cases/README.md new file mode 100644 index 0000000..21e31c0 --- /dev/null +++ b/solutions/use_cases/README.md @@ -0,0 +1,77 @@ + 🚀 Use Cases Examples 🚀 +=============== + +**End-to-End Scenarios** + +In this repository, we feature several 'end-to-end' examples that show how to use LLMWare in a complex recipe combining different elements to accomplish a specific objective. While each example is still high-level, it is shared in the spirit of providing a high-level framework 'starting point' that can be developed in more detail for a variety of common use cases. All of these examples use small, specialized models, running locally - 'Small, but Mighty' ! + + +1. [**Research Automation with Agents and Web Services**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/web_services_slim_fx.py) + + - Prepare a 30-key research analysis on a company + - Extract key lookup and other information from an earnings press release + - Automatically use the lookup data for real-time stock information from YFinance + - Automatically use the lookup date for background company history information in Wikipedia + - Run LLM prompts to ask key questions of the Wikipedia sources + - Aggregate into a consolidated research analysis + - All with local open source models + + +2. [**Invoice Processing**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/invoice_processing.py) + + - Parse a batch of invoices (provided as sample files) + - Extract key information from the invoices + - Save the prompt state for follow-up review and analysis + + +3. [**Analyzing and Extracting Voice Transcripts**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/parsing_great_speeches.py) + + - Voice transcription of 50+ wav files of great speeches of the 20th century + - Run text queries against the transcribed wav files + - Execute LLM agent inferences to extract and identify key elements of interest + - Prepare 'bibliography' with the key extracted points, including time-stamp + + +4. [**MSA Processing**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/msa_processing.py) + + - Identify the termination provisions in Master Service Agreements among a larger batch of contracts + - Parse and query a large batch of contracts and identify the agreements with "Master Service Agreement" on the first page + - Find the termination provisions in each MSA + - Prompt LLM to read the termination provisions and answer a key question + - Run a fact-check and source-check on the LLM response + - Save all of the responses in CSV and JSON for follow-up review. + + +5. [**Querying a CSV**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/agent_with_custom_tables.py) + + - Start running natural language queries on CSVs with Postgres and slim-sql-tool. + - Load a sample 'customer_table.csv' into Postgres + - Start running natural language queries that get converted into SQL and query the DB + + +6. [**Contract Analysis**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/contract_analysis_on_laptop_with_bling_models.py) + + - Extract key information from set of employment agreement + - Use a simple retrieval strategy with keyword search to identify key provisions and topic areas + - Prompt LLM to read the key provisions and answer questions based on those source materials + +7. [**Slicing and Dicing Office Docs**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/slicing_and_dicing_office_docs.py) + + - Shows a variety of advanced parsing techniques with Office document formats packaged in ZIP archives + - Extracts tables and images, runs OCR against the embedded images, exports the whole library, and creates dataset + +8. **LLMWare Private Inference Server** + + - Set up server in minutes on CPU, GPU or local - [server](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/llmware_inference_server.py) + + - Run 3 different modes of client access to the API - [client](https://www.github.com/llmware-ai/llmware/tree/main/examples/Use_Cases/llmware_inference_api_client.py) + + - Supports rapid development, testing and prototyping and flexibility of deployment models for wide range of RAG and Agent use cases + + +Check back often - we are updating these examples regularly - and many of these examples have companion videos as well. + + +### **Let's get started! 🚀** + + diff --git a/solutions/use_cases/agent_with_custom_tables.py b/solutions/use_cases/agent_with_custom_tables.py new file mode 100644 index 0000000..65eea0f --- /dev/null +++ b/solutions/use_cases/agent_with_custom_tables.py @@ -0,0 +1,118 @@ + +""" This example shows an end-to-end recipe for creating a CustomTable, and then creating an Agent process that + will query the table using natural language. + + Please note that this example is a 'generalized' and updated version of an earlier example - + "text2sql-end-to-end-2.py" - now using the more powerful CustomTables class integrated into the LLMfx process + + The example shows the following steps: + + 1. Creating a custom table resource from a sample CSV file, included in the slim-sql-tool kit, and also + available in the Examples section with Structured_Tables (customer_table.csv) + + 2 Asking basic natural language questions: + A. Looks up the table schema + B. Packages the table schema with query + C. Runs inference to convert text into SQL + D. Queries the database with the generated SQL + E. Returns result + + 3. Using CustomtTable class, this can be run on either Postgres or SQLite DB. + + Note: as you substitute for your own CSV and JSON, check out the other examples in this section for loading + configuration ideas and options. + +""" + +import os + +from llmware.agents import LLMfx +from llmware.resources import CustomTable +from llmware.configs import LLMWareConfig + + +def build_table(db=None, table_name=None,load_fp=None,load_file=None): + + """ Simple example script to take a CSV or JSON/JSONL and create a DB Table. """ + + custom_table = CustomTable(db=db, table_name=table_name) + analysis = custom_table.validate_csv(load_fp, load_file) + print("update: analysis from validate_csv: ", analysis) + + if load_file.endswith(".csv"): + output = custom_table.load_csv(load_fp, load_file) + elif load_file.endswith(".jsonl") or load_file.endswith(".json"): + output = custom_table.load_json(load_fp, load_file) + else: + print("file type not supported for db load") + return -1 + + print("update: output from loading file: ", output) + + sample_range = min(10, len(custom_table.rows)) + for x in range(0,sample_range): + print("update: sample rows: ", x, custom_table.rows[x]) + + # stress the schema data type and remediate - use more samples for more accuracy + updated_schema = custom_table.test_and_remediate_schema(samples=20, auto_remediate=True) + + print("update: updated schema: ", updated_schema) + + # insert the rows in the DB + custom_table.insert_rows() + + return 1 + + +def agent_natural_language_sql_query(query_list, db=None, table_name=None): + + """ Query a CustomTable in natural language. """ + + # query starts here + agent = LLMfx() + agent.load_tool("sql", sample=False, get_logits=True, temperature=0.0) + + # Pass direct queries to the DB + + for i, query in enumerate(query_list): + + # query_custom_table method is doing all of the work + # -- looks up the table schema in the db using the table_name + # -- packages the text-2-sql query prompt + # -- executes sql method to convert the prompt into a sql query + # -- attempts to execute the sql query on the db + # -- returns the db results as 'research' output + + response = agent.query_custom_table(query,db=db,table=table_name) + + for x in range(0,len(agent.research_list)): + print("research: ", x, agent.research_list[x]) + + return agent.research_list + + +if __name__ == "__main__": + + # input parameters - db, table_name, path to csv or json/jsonl file + db = "postgres" + table_name = "customer_table" + + # for hello_world, please pull down the customer_table.csv found in the examples repository + input_fp = "/local/path/to/csv/" + input_fn = "customer_table.csv" + + # builds table - only needs to be done once + build_table(db=db, table_name=table_name, load_fp=input_fp, load_file=input_fn) + + # list of queries to ask the csv or json/jsonl + # -- note: these queries are useful but purposefully straightforward with relatively clear alignment + # to the data schema + + query_list = ["Which customers have vip customer status of yes?", + "What is the highest annual spend of any customer?", + "Which customer has account number 1234953", + "Which customer has the lowest annual spend?", + "Is Susan Soinsin a vip customer?"] + + # agent process to execute the query list + agent_natural_language_sql_query(query_list, db=db, table_name=table_name) diff --git a/solutions/use_cases/biz_bot.py b/solutions/use_cases/biz_bot.py new file mode 100644 index 0000000..311b6d7 --- /dev/null +++ b/solutions/use_cases/biz_bot.py @@ -0,0 +1,371 @@ + +""" This example provides a basic framework to build a "Biz Bot" designed to integrate both 'RAG' source documents +and 'SQL' tables (e.g., CSV files), and interface using natural language with a chatbot UI. + + Models - this example uses 3 models, running locally + + -- bling-phi-3-gguf - core RAG question-answer model + -- slim-sql-tool - fast local text-to-sql model + -- jina-reranker-turbo - reranking model from Jina AI + -- for more info on Jina AI model - see https://huggingface.co/jinaai/jina-reranker-v1-turbo-en + + Database - the SQL functionality will load a table into SQLite (no install required) + + Pre-reqs - + + 1. Streamlit - to run this example requires an install of Streamlit, e.g., `pip3 install streamlit` + + -- To execute the script, run from the command line with: `streamlit run biz_bot.py` + -- For more information on the Streamlit Chat UI, see + https://docs.streamlit.io/develop/tutorials/llms/build-conversational-apps + + Sample Data - + + -- by default, at startup, the app will load a sample Employment Agreement document and a simple customer table + + -- RAG test question: 'what is the annual rate of the base salary?' + -- SQL test question: 'which customer has the lowest annual spend?' + + 2. Add new documents or CSV files on the left side panel upload buttons. + + All components of the Biz Bot will be running locally, so the speed will be determined greatly by the + CPU/GPU capacities and ** memory ** of your machine. We would recommend at least 16 GB of RAM, and ideally 32 GB + to run this example. + +""" + +import os + +import streamlit as st +from llmware.resources import CustomTable +from llmware.models import ModelCatalog +from llmware.prompts import Prompt +from llmware.parsers import Parser +from llmware.configs import LLMWareConfig +from llmware.agents import LLMfx +from llmware.setup import Setup + +# keeps a running state of any csv tables that have been loaded in the session to avoid duplicated inserts +if "loaded_tables" not in st.session_state: + st.session_state["loaded_tables"] = [] + + +def build_table(db=None, table_name=None,load_fp=None,load_file=None): + + """ Simple example script to take a CSV or JSON/JSONL and create a DB Table. + + """ + + if not table_name: + return 0 + + # build the table only once - if name already found, do not add to table + if table_name in st.session_state["loaded_tables"]: + return 0 + + custom_table = CustomTable(db=db, table_name=table_name) + analysis = custom_table.validate_csv(load_fp, load_file) + print("update: analysis from validate_csv: ", analysis) + + if load_file.endswith(".csv"): + output = custom_table.load_csv(load_fp, load_file) + elif load_file.endswith(".jsonl") or load_file.endswith(".json"): + output = custom_table.load_json(load_fp, load_file) + else: + print("file type not supported for db load") + return -1 + + print("update: output from loading file: ", output) + + sample_range = min(10, len(custom_table.rows)) + for x in range(0,sample_range): + print("update: sample rows: ", x, custom_table.rows[x]) + + # stress the schema data type and remediate - use more samples for more accuracy + updated_schema = custom_table.test_and_remediate_schema(samples=20, auto_remediate=True) + + print("update: updated schema: ", updated_schema) + + # insert the rows in the DB + custom_table.insert_rows() + + st.session_state["loaded_tables"].append(table_name) + + return len(custom_table.rows) + + +@st.cache_resource +def load_reranker_model(): + + """ Loads the reranker model used in the RAG process to rank the semantic similarity between all of the + parsed text chunks and the user query. """ + + reranker_model = ModelCatalog().load_model("jina-reranker-turbo") + return reranker_model + + +@st.cache_resource +def load_prompt_model(): + + """ Loads the core RAG model used for fact-based question-answering against the source materials. """ + + prompter = Prompt().load_model("bling-phi-3-gguf", temperature=0.0, sample=False) + return prompter + + +@st.cache_resource +def load_agent_model(): + + """ Loads the Text2SQL model used for querying the CSV table. """ + + agent = LLMfx() + agent.load_tool("sql", sample=False, get_logits=True, temperature=0.0) + return agent + + +@st.cache_resource +def parse_file(fp, doc): + + """ Executes a parsing job of a newly uploaded file, and saves the parser out as a set of text chunks + with metadata. """ + + parser_output = Parser().parse_one(fp, doc, save_history=False) + st.cache_resource.clear() + return parser_output + + +def get_rag_response(prompt, parser_output, reranker_model, prompter): + + """ Executes a RAG response. """ + + # if the number of text chunks is small, then will skip the reranker + if len(parser_output) > 3: + output = reranker_model.inference(prompt, parser_output, top_n=10, relevance_threshold=0.25) + else: + output = [] + for entries in parser_output: + entries.update({"rerank_score": 0.0}) + output.append(entries) + + use_top = 3 + if len(output) > use_top: + output = output[0:use_top] + + # create the source from the top 3 text chunks + sources = prompter.add_source_query_results(output) + + # run the inference on the model with the source automatically packaged and attached + responses = prompter.prompt_with_source(prompt, prompt_name="default_with_context") + + # execute post-inference fact and source checking + source_check = prompter.evidence_check_sources(responses) + numbers_check = prompter.evidence_check_numbers(responses) + nf_check = prompter.classify_not_found_response(responses,parse_response=True,evidence_match=False,ask_the_model=False) + + bot_response = "" + for i, resp in enumerate(responses): + bot_response = resp['llm_response'] + + print("bot response - llm_response raw - ", bot_response) + + add_sources = True + + if "not_found_classification" in nf_check[i]: + + if nf_check[i]["not_found_classification"]: + add_sources = False + bot_response += "\n\n" + ("The answer to the question was not found in the source " + "passage attached - please check the source again, and/or " + "try to ask the question in a different way.") + + if add_sources: + numbers_output = "" + if "fact_check" in numbers_check[i]: + fc = numbers_check[i]["fact_check"] + if isinstance(fc, list) and len(fc) > 0: + + max_fact_count = 1 + count = 0 + + for fc_entries in fc: + + if count < max_fact_count: + + if "text" in fc_entries: + numbers_output += "Text: " + fc_entries["text"] + "\n\n" + if "source" in fc_entries: + numbers_output += "Source: " + fc_entries["source"] + "\n\n" + if "page_num" in fc_entries: + numbers_output += "Page Num: " + fc_entries["page_num"] + "\n\n" + count += 1 + + bot_response += "\n\n" + numbers_output + + source_output = "" + if not numbers_output: + if "source_review" in source_check[i]: + fc = source_check[i]["source_review"] + if isinstance(fc, list) and len(fc) > 0: + fc = fc[0] + if "text" in fc: + source_output += "Text: " + fc["text"] + "\n\n" + if "match_score" in fc: + source_output += "Match Score: " + str(fc["match_score"]) + "\n\n" + if "source" in fc: + source_output += "Source: " + fc["source"] + "\n\n" + if "page_num" in fc: + source_output += "Page Num: " + str(fc["page_num"]) + "\n\n" + + bot_response += "\n\n" + source_output + + prompter.clear_source_materials() + + return bot_response + + +def get_sql_response(prompt, agent, db=None, table_name=None): + + """ Executes a Text-to-SQL inference, and then also queries the database and returns the database query result. """ + + # note: there is an optional magic word " #SHOW" at the end of a user query, which is stripped from the query + # before generating the SQL, but is then used by the UI to display the SQL command produced. + + show_sql = False + bot_response = "" + + if prompt.endswith(" #SHOW"): + show_sql = True + prompt = prompt[:-(len(" #SHOW"))] + + model_response = agent.query_custom_table(prompt, db=db, table=table_name) + + # insert additional error checking / post-processing of output here + error_handle = False + + try: + sql_query = model_response["sql_query"] + db_response = model_response["db_response"] + + if not show_sql: + bot_response = db_response + else: + bot_response = f"Answer: {db_response}\n\nSQL Query: {sql_query}" + except: + error_handle = True + sql_query = None + db_response = None + + if error_handle or not sql_query or not db_response: + bot_response = (f"Sorry I could not find an answer to your question.
" + f"Here is the SQL query that was generated by your question: " + f"
{sql_query}.
If this missed the mark, please try asking " + f"the question again with a little more specificity.") + + return bot_response + + +def biz_bot_ui_app (db="postgres", table_name=None, fp=None,doc=None): + + st.title(f"Biz Bot") + + parser_output = None + + if os.path.exists(os.path.join(fp,doc)): + if not parser_output: + parser_output = Parser().parse_one(fp, doc, save_history=False) + + prompter = load_prompt_model() + reranker_model = load_reranker_model() + agent = load_agent_model() + + # initialize chat history + if "messages" not in st.session_state: + st.session_state.messages = [] + + # display chat messages from history on app rerun + for message in st.session_state.messages: + with st.chat_message(message["role"]): + st.markdown(message["content"]) + + with st.sidebar: + + st.write("Biz Bot") + model_type = st.selectbox("Pick your mode", ("RAG","SQL"), index=0) + + uploaded_doc = st.file_uploader("Upload Document") + uploaded_table = st.file_uploader("Upload CSV") + + if uploaded_doc: + fp = LLMWareConfig().get_llmware_path() + doc = uploaded_doc.name + f = open(os.path.join(fp,doc), "wb") + f.write(uploaded_doc.getvalue()) + f.close() + parser_output = parse_file(fp, doc) + st.write(f"Document Parsed and Ready - {len(parser_output)}") + + if uploaded_table: + fp = LLMWareConfig().get_llmware_path() + tab = uploaded_table.name + f = open(os.path.join(fp,tab),"wb") + f.write(uploaded_table.getvalue()) + f.close() + table_name = tab.split(".")[0] + st.write("Building Table - ", tab, table_name) + st.write(st.session_state['loaded_tables']) + row_count = build_table(db=db, table_name=table_name, load_fp=fp, load_file=tab) + st.write(f"Completed - Table - {table_name} - Rows - {row_count} - is Ready.") + + # accept user input + prompt = st.chat_input("Say something") + if prompt: + + with st.chat_message("user"): + st.markdown(prompt) + + with (st.chat_message("assistant")): + + if model_type == "RAG": + bot_response = get_rag_response(prompt,parser_output, reranker_model, prompter) + st.markdown(bot_response) + else: + bot_response = get_sql_response(prompt, agent, db=db, table_name=table_name) + st.markdown(bot_response) + + st.session_state.messages.append({"role": "user", "content": prompt}) + st.session_state.messages.append({"role": "assistant", "content": bot_response}) + + return 0 + + +if __name__ == "__main__": + + # note: there is a hidden 'magic' command in the chatbot - if you add " #SHOW" at the end of your query, + # then it will display the SQL command that was generated (very useful for debugging) + + db = "sqlite" + table_name = "customer_table_1" + + # By default, the BizBot starts with a loaded document and CSV (which can then be changed directly in the UI) + + # pull customer_table.csv sample file from the same repo as this example, or alternatively + # substitute your own csv to get started + + local_csv_path = "/your/local/path/to/csv/" + build_table(db=db, table_name=table_name,load_fp=local_csv_path, load_file="customer_table.csv") + + # get sample agreement to use as a starting point or feel free to substitute your own document + + sample_files_path = Setup().load_sample_files(over_write=False) + fp = os.path.join(sample_files_path, "Agreements") + fn = "Nike EXECUTIVE EMPLOYMENT AGREEMENT.pdf" + + biz_bot_ui_app(db=db, table_name=table_name,fp=fp, doc=fn) + + # IMPORTANT NOTE: to run this script, follow the streamlit formalism and run from the cli, e.g., + # `streamlit run biz_bot.py` + + + + + diff --git a/solutions/use_cases/contract_analysis_on_laptop_with_bling_models.py b/solutions/use_cases/contract_analysis_on_laptop_with_bling_models.py new file mode 100644 index 0000000..91344b1 --- /dev/null +++ b/solutions/use_cases/contract_analysis_on_laptop_with_bling_models.py @@ -0,0 +1,67 @@ + +"""This example demonstrates a basic contract analysis workflow run entirely on on a laptop + using a new Bling-Phi-3 RAG-finetuned small specialized instruct model. +""" + +import os +import re +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.setup import Setup +from llmware.configs import LLMWareConfig + + +def contract_analysis_on_laptop (model_name): + + # Load the llmware sample files + print (f"\n > Loading the llmware sample files...") + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path,"Agreements") + + # query list + query_list = {"executive employment agreement": "What are the name of the two parties?", + "base salary": "What is the executive's base salary?", + "vacation": "How many vacation days will the executive receive?"} + + print (f"\n > Loading model {model_name}...") + + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + + for i, contract in enumerate(os.listdir(contracts_path)): + + # exclude potential mac os created file artifact in folder path + if contract != ".DS_Store": + + print("\nAnalyzing contract: ", str(i+1), contract) + + print("LLM Responses:") + + for key, value in query_list.items(): + + # contract is parsed, text-chunked, and then filtered by topic key + source = prompter.add_source_document(contracts_path, contract, query=key) + + # calling the LLM with 'source' information from the contract automatically packaged into the prompt + responses = prompter.prompt_with_source(value, prompt_name="default_with_context") + + for r, response in enumerate(responses): + print(key, ":", re.sub("[\n]"," ", response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + + # Save jsonl report to jsonl to /prompt_history folder + print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + prompter.save_state() + + #Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + print("csv output - ", csv_output) + + +if __name__ == "__main__": + + # use new bling phi-3 model quantized and running on local cpu + model = "bling-phi-3-gguf" + + contract_analysis_on_laptop(model) + diff --git a/solutions/use_cases/invoice_processing.py b/solutions/use_cases/invoice_processing.py new file mode 100644 index 0000000..233de11 --- /dev/null +++ b/solutions/use_cases/invoice_processing.py @@ -0,0 +1,103 @@ + +""" This example shows an end-to-end scenario for invoice processing that can be run locally and without a +database. The example shows how to combine the use of parsing combined with prompts_with_sources to rapidly +iterate through a batch of invoices and ask a set of questions, and then save the full output to both +(1) .jsonl for integration into an upstream application/database and (2) to a CSV for human review in excel. + + note: the sample code pulls from a public repo to load the sample invoice documents the first time - + please feel free to substitute with your own invoice documents (PDF/DOCX/PPTX/XLSX/CSV/TXT) if you prefer. + + this example does not require a database or embedding + + this example can be run locally on a laptop by setting 'run_on_cpu=True' + if 'run_on_cpu==False", then please see the example 'launch_llmware_inference_server.py' + to configure and set up a 'pop-up' GPU inference server in just a few minutes +""" + +import os +import re + +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.configs import LLMWareConfig +from llmware.setup import Setup +from llmware.models import ModelCatalog + + +def invoice_processing(run_on_cpu=True): + + # Step 1 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + invoices_path = os.path.join(sample_files_path, "Invoices") + + # Step 2 - simple sample query list - each question will be asked to each invoice + query_list = ["What is the total amount of the invoice?", + "What is the invoice number?", + "What are the names of the two parties?"] + + # Step 3 - Load Model + + if run_on_cpu: + + # load local bling model that can run on cpu/laptop + + # note: bling-1b-0.1 is the *fastest* & *smallest*, but will make more errors than larger BLING models + # model_name = "llmware/bling-1b-0.1" + + # try the new bling-phi-3 quantized with gguf - most accurate + model_name = 'bling-phi-3-gguf' + else: + + # use GPU-based inference server to process + # *** see the launch_llmware_inference_server.py example script to setup *** + + server_uri_string = "http://11.123.456.789:8088" # insert your server_uri_string + server_secret_key = "demo-test" + ModelCatalog().setup_custom_llmware_inference_server(server_uri_string, secret_key=server_secret_key) + model_name = "llmware-inference-server" + + # attach inference server to prompt object + prompter = Prompt().load_model(model_name) + + # Step 4 - main loop thru folder of invoices + + for i, invoice in enumerate(os.listdir(invoices_path)): + + # just in case (legacy on mac os file system - not needed on linux or windows) + if invoice != ".DS_Store": + + print("\nAnalyzing invoice: ", str(i + 1), invoice) + + for question in query_list: + + # Step 4A - parses the invoices in memory and attaches as a source to the Prompt + source = prompter.add_source_document(invoices_path,invoice) + + # Step 4B - executes the prompt on the LLM (with the loaded source) + output = prompter.prompt_with_source(question,prompt_name="default_with_context") + + for i, response in enumerate(output): + print("LLM Response - ", question, " - ", re.sub("[\n]"," ", response["llm_response"])) + + prompter.clear_source_materials() + + # Save jsonl report with full transaction history to /prompt_history folder + print("\nupdate: prompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + + prompter.save_state() + + # Generate CSV report for easy Human review in Excel + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + + print("\nupdate: csv output for human review - ", csv_output) + + return 0 + + +if __name__ == "__main__": + + invoice_processing(run_on_cpu=True) + + diff --git a/solutions/use_cases/lecture_tool/Home.py b/solutions/use_cases/lecture_tool/Home.py new file mode 100644 index 0000000..34a2c4e --- /dev/null +++ b/solutions/use_cases/lecture_tool/Home.py @@ -0,0 +1,65 @@ +import streamlit as st + + +home_text = ''' +# Lecture Tool + +Welcome to Lecture Tool! This is an AI application that leverages `llmware` to \ +transcribe and analyze college lecture videos. + +--- + +### Features +1. Create `Libraries` to group transcripts and store them persistently. +2. Transcribe audio files using Whisper (built into the `llmware` library) and \ +store them in a `Library`. +3. Ask general questions and questions about lecture content. +4. Summarize lecture content. +5. View all generated transcripts. + +Each of these five features is implemented in its own file in the `pages` \ +folder. + +--- + +### Prerequisites +1. *Python libraries*: the only required libraries to be installed are \ +`streamlit` and `llmware`. You can install them from the `requirements.txt` \ +file. +2. *MongoDB*: it is used to store lecture transcripts. The easiest way to \ +install it is to use the Docker Compose file in the \ +[LLMWare repository](https://github.com/llmware-ai/llmware/blob/main/docker-compose_mongo_milvus.yaml). +3. *FFmpeg*: it is used to convert MP3 files to WAV files that are compatible \ +with Whipser. If you intend to use MP3 files instead of WAV files, you can \ +[download FFmpeg here](https://www.ffmpeg.org/download.html). You will likely \ +need to restart your computer after installation. + +--- + +### Usage +To run the program, ensure that you have `streamlit` installed. In your \ +terminal, navigate to the `lecture_tool` directory and run `streamlit run \ +Home.py`. + +By default, Streamlit supports file uploads up to 200 MB. To increase \ +this limit, run `streamlit run Home.py --server.maxUploadSize fileSize`, \ +ensuring to replace `fileSize` with the maximum file size you want to upload \ +in megabytes. For example, use 500 if you plan to upload audio files up to 500 \ +MB in size. + +Sample MP3 and WAV audio files to use the application with are available in \ +the `sample_audio_files` directory. + +The `saved_files` directory is used as a temporary location in the \ +application's implementation and should not be modified by a user. +''' + +# +# Launches the home page of the program. + +# Run `streamlit run Home.py` to start the application. +# +if __name__ == '__main__': + st.sidebar.write("Select a page above.") + + st.write(home_text) diff --git a/solutions/use_cases/lecture_tool/README.md b/solutions/use_cases/lecture_tool/README.md new file mode 100644 index 0000000..a3ba8d5 --- /dev/null +++ b/solutions/use_cases/lecture_tool/README.md @@ -0,0 +1,32 @@ +# Lecture Tool + +Welcome to Lecture Tool! This is an AI application that leverages `llmware` to transcribe and analyze college lecture videos. + +--- + +### Features +1. Create `Libraries` to group transcripts and store them persistently. +2. Transcribe audio files using Whisper (built into the `llmware` library) and store them in a `Library`. +3. Ask general questions and questions about lecture content. +4. Summarize lecture content. +5. View all generated transcripts. + +Each of these five features is implemented in its own file in the `pages` folder. + +--- + +### Prerequisites +1. *Python libraries*: the only required libraries to be installed are `streamlit` and `llmware`. You can install them from the `requirements.txt` file. +2. *MongoDB*: it is used to store lecture transcripts. The easiest way to install it is to use the Docker Compose file in the [LLMWare repository](https://github.com/llmware-ai/llmware/blob/main/docker-compose_mongo_milvus.yaml). +3. *FFmpeg*: it is used to convert MP3 files to WAV files that are compatible with Whipser. If you intend to use MP3 files instead of WAV files, you can [download FFmpeg here](https://www.ffmpeg.org/download.html). You will likely need to restart your computer after installation. + +--- + +### Usage +To run the program, ensure that you have `streamlit` installed. In your terminal, navigate to the `lecture_tool` directory and run `streamlit run Home.py`. + +By default, Streamlit supports file uploads up to 200 MB. To increase this limit, run `streamlit run Home.py --server.maxUploadSize fileSize`, ensuring to replace `fileSize` with the maximum file size you want to upload in megabytes. For example, use 500 if you plan to upload audio files up to 500 MB in size. + +Sample MP3 and WAV audio files to use the application with are available in the `sample_audio_files` directory. + +The `saved_files` directory is used as a temporary location in the application's implementation and should not be modified by a user. diff --git a/solutions/use_cases/lecture_tool/Utils.py b/solutions/use_cases/lecture_tool/Utils.py new file mode 100644 index 0000000..50d4f7c --- /dev/null +++ b/solutions/use_cases/lecture_tool/Utils.py @@ -0,0 +1,33 @@ +from llmware.library import Library +from llmware.retrieval import Query + + +ACCOUNT_NAME = 'lecture_tool' + + +""" +Creates a list of all libraries created with the application based on the +ACCOUNT_NAME. +""" +def get_stored_libraries(): + all_library_cards = Library().get_all_library_cards(account_name=ACCOUNT_NAME) + + library_list = [] + for card in all_library_cards: + library_list.append(card['library_name']) + + return library_list + + +""" +Creates a list of unique filenames in a specified library. +""" +def get_stored_files(library_name): + library = Library().load_library(library_name, account_name=ACCOUNT_NAME) + + file_list = [] + for library_info in Query(library).get_whole_library(): + if library_info['file_source'] not in file_list: + file_list.append(library_info['file_source']) + + return file_list diff --git a/solutions/use_cases/lecture_tool/pages/1_Manage_libraries.py b/solutions/use_cases/lecture_tool/pages/1_Manage_libraries.py new file mode 100644 index 0000000..3b95636 --- /dev/null +++ b/solutions/use_cases/lecture_tool/pages/1_Manage_libraries.py @@ -0,0 +1,66 @@ +from llmware.library import Library + +import streamlit as st + +import os +import sys + +sys.path.insert(0, os.getcwd()) + +from Utils import get_stored_libraries + + +ACCOUNT_NAME = 'lecture_tool' + + +# +# Creates an llmware Library. +# +def create_library(library_name): + Library().create_new_library(library_name, account_name=ACCOUNT_NAME) + print('\nupdate: library created - ', library_name) + + +# +# Deletes an existing library if it exists. +# +def delete_library(library_name): + card = Library().check_if_library_exists(library_name, account_name=ACCOUNT_NAME) + + if card: + Library().delete_library(library_name, confirm_delete=True, account_name=ACCOUNT_NAME) + print('\nupdate: delete library - ', library_name) + else: + print('\nupdate: library does not exist - ', library_name) + + +# +# Main block for GUI logic. +# +if __name__ == '__main__': + st.title('Manage your lecture libraries') + + st.write('### Delete library/libraries') + + stored_libraries = get_stored_libraries() + library_names = st.multiselect( + 'Select library/libraries to delete:', + stored_libraries, + max_selections=len(stored_libraries) + ) + + if st.button('Delete'): + for library_name in library_names: + delete_library(library_name) + + st.success(f'Successfully deleted {library_name}') + + st.write('### Create library') + + library_name = st.text_input('Enter library name to create') + + if st.button('Create'): + if library_name: + create_library(library_name) + + st.success(f'Successfully created {library_name}') diff --git a/solutions/use_cases/lecture_tool/pages/2_Manage_files.py b/solutions/use_cases/lecture_tool/pages/2_Manage_files.py new file mode 100644 index 0000000..753d25d --- /dev/null +++ b/solutions/use_cases/lecture_tool/pages/2_Manage_files.py @@ -0,0 +1,73 @@ +from llmware.parsers import Parser +from llmware.library import Library + +import streamlit as st + +import os +import sys + +sys.path.insert(0, os.getcwd()) + +from Utils import get_stored_libraries + + +SAVED_FILES_WD = os.path.join(os.getcwd(), 'saved_files') +ACCOUNT_NAME = 'lecture_tool' + + +# +# Deletes existing temporary files in saved_files directory. +# +def delete_all_saved_files(): + for file in os.listdir(SAVED_FILES_WD): + os.remove(os.path.join(SAVED_FILES_WD, file)) + print('\nupdate: deleted temporary file - ', file) + + +# +# Uses Parser class to transcribe audio files uploaded and stores the output to +# the specified library. +# +def parse_lecture_file_and_store(filename, library_name): + library = Library().load_library(library_name, account_name=ACCOUNT_NAME) + + parser_output = Parser(chunk_size=400, max_chunk_size=600, library=library).parse_voice(SAVED_FILES_WD, real_time_progress=True, copy_to_library=True) + print('\nupdate: parser output - ', parser_output) + + print('\nupdate: library card - ', library.get_library_card()) + + os.remove(os.path.join(SAVED_FILES_WD, filename)) + print('\nupdate: deleted temporary file - ', filename) + + +# +# Main block for GUI logic. +# +if __name__ == '__main__': + st.title('Manage your lecture files') + + st.write('### Select library') + + stored_libraries = get_stored_libraries() + library_name = st.selectbox( + 'Select library to manage files:', + stored_libraries + ) + + st.write('### Add file(s)') + + files = st.file_uploader('Upload audio file(s)', type=['wav', 'mp3'], accept_multiple_files=True) + + if st.button('Upload'): + delete_all_saved_files() + + for file in files: + if file is not None: + with open(os.path.join(SAVED_FILES_WD, file.name), 'wb') as f: + f.write(file.getbuffer()) + print('\nupdate: added file to temporary storage - ', file.name) + + with st.spinner(f'Processing file {file.name}... don\'t leave this page!'): + parse_lecture_file_and_store(file.name, library_name) + + st.success(f'Successfully stored {file.name}') diff --git a/solutions/use_cases/lecture_tool/pages/3_Prompts.py b/solutions/use_cases/lecture_tool/pages/3_Prompts.py new file mode 100644 index 0000000..7294c71 --- /dev/null +++ b/solutions/use_cases/lecture_tool/pages/3_Prompts.py @@ -0,0 +1,205 @@ +from llmware.gguf_configs import GGUFConfigs +from llmware.library import Library +from llmware.prompts import Prompt +from llmware.retrieval import Query +from llmware.models import ModelCatalog + +import streamlit as st + +import os +import sys + +sys.path.insert(0, os.getcwd()) + +from Utils import get_stored_files, get_stored_libraries + +GENERAL_PURPOSE_MODEL = 'phi-3-gguf' +RAG_MODEL = 'bling-phi-3-gguf' +RERANKER_MODEL = 'jina-reranker-turbo' + +ACCOUNT_NAME = 'lecture_tool' + + +# +# Calls the GENERAL_PURPOSE_MODEL defined above on the specified question. +# +@st.cache_data(show_spinner=False) +def prompt_general_question(question): + # Set limit on output length + GGUFConfigs().set_config("max_output_tokens", 1000) + + # Load in appropriate model + prompter = Prompt().load_model(GENERAL_PURPOSE_MODEL, max_output=999) + + # Prompt the model with the question + print('\nupdate: performing prompt') + response = prompter.prompt_main(question) + print('\nupdate: llm response - ', response) + + return response['llm_response'] + + +# +# Calls the RAG_MODEL defined above to answer questions about lecture content. +# +# Accesses the library_name specified to pass either the entire library or a +# specified file from the library as source for reranking. +# +# RERANKER_MODEL defined above generates a list of the most relevant text blocks +# based on the question (prompt) with semantic reranking. + +# These are passed as source to the RAG_MODEL for inference along with the +# question (prompt). +# +@st.cache_data(show_spinner=False) +def prompt_question_about_content_with_reranking(question, library_name, filename, topic=None): + # Load in appropriate library + library = Library().load_library(library_name, account_name=ACCOUNT_NAME) + print('\nupdate: library card - ', library.get_library_card()) + + # Create Query object + query = Query(library) + + # Load in appropriate models + rag_prompter = Prompt().load_model(RAG_MODEL, temperature=0.0, sample=False) + reranker_model = ModelCatalog().load_model(RERANKER_MODEL, temperature=0.0, sample=False) + + # Access appropriate text blocks if all files are to be selected + if filename == 'Select all files': + print('\nupdate: all files selected') + + # Access entire library since no topic is specified + if topic is None or topic == '': + print('\nupdate: no topic provided, adding entire library as source') + query_results = query.get_whole_library() + + # Change key in query results for compatibility with RAG call + for result in query_results: + result['text'] = result['text_search'] + del result['text_search'] + + print('\nupdate: correct library chunks - ', query_results) + # Access limited blocks from library based on topic specified + else: + # Perform text_query with the topic + print('\nupdate: topic provided, performing text query for topic') + query_results = query.text_query(topic) + print('\nupdate: topic chunks - ', query_results) + # Access appropriate text chunks if a specific file is selected + else: + print('\nupdate: file selected - ', filename) + + # Access all blocks for the specified file since no topic is specified + if topic is None or topic == '': + print('\nupdate: no topic provided, adding entire library as source') + + # Filter out only the blocks that correspond to the desired file + query_results = query.apply_custom_filter(query.get_whole_library(), {'file_source': filename}) + + # Change key in query results for compatibility with RAG call + for result in query_results: + result['text'] = result['text_search'] + del result['text_search'] + + print('\nupdate: correct library chunks - ', query_results) + # Access limited blocks from library based on topic and file specified + else: + # Perform a text query for the topic, then filter based on filename + print('\nupdate: topic provided, performing text query for topic') + query_results = query.apply_custom_filter(query.text_query(topic), {'file_source': filename}) + print('\nupdate: topic chunks - ', query_results) + + if len(query_results) == 0: + print('\nupdate: sources are empty') + return 'No result found. Please check to ensure your topic and file are accurate.', None, None + + # Perform semantic reranking to get relevant text blocks + print('\nupdate: performing semantic ranking') + reranker_output = reranker_model.inference(question, query_results, top_n=10, relevance_threshold=0.2) + + # Use only the 3 most relevant blocks if more are returned from rereanking + use_top = 3 + if len(reranker_output) > use_top: + reranker_output = reranker_output[:use_top] + + print('\nupdate: reranker output - ') + for i, source in enumerate(reranker_output): + print(i, ' - ', source) + + # Pass relevant blocks from reranking as source for RAG call + sources = rag_prompter.add_source_query_results(reranker_output) + print('\nupdate: sources - ', sources) + + # Perform RAG call with the question + print('\nupdate: performing prompt') + response = rag_prompter.prompt_with_source(question) + + # Store metadata about the RAG call for later use + stats = rag_prompter.evidence_comparison_stats(response) + ev_source = rag_prompter.evidence_check_sources(response) + + # Output LLM response + for i, resp in enumerate(response): + print('\nupdate: llm response - ', resp) + print('\nupdate: compare with evidence - ', stats[i]['comparison_stats']) + print('\nupdate: sources - ', ev_source[i]['source_review']) + + # Clear the source so the next call does not reuse previous source + rag_prompter.clear_source_materials() + + source_file = response[0]['source_review'][0]['source'] if 'source_review' in response[0] and len(response[0]['source_review']) > 0 else None + return response[0]['llm_response'], response[0]['evidence'], source_file + + +# +# Main block for GUI logic. +# +if __name__ == '__main__': + st.title('Ask questions about lecture content') + + st.write('### Question info') + + question_type = st.selectbox( + 'Select the type of question:', + ('Question about Content', 'General Question') + ) + + question = st.text_input('Enter your question:') + + if question_type == 'Question about Content': + st.write('### Prompt info') + + topic = st.text_input('Optionally enter a topic:') + + library_name = st.selectbox( + 'Select the library:', + tuple(get_stored_libraries()) + ) + + if library_name: + filename = st.selectbox( + 'Select the file:', + tuple(['Select all files'] + get_stored_files(library_name)) + ) + + if st.button('Prompt'): + if question_type == 'General Question': + with st.spinner('Processing request... don\'t leave this page!'): + response = prompt_general_question(question) + + st.write('### Response') + st.write(response) + elif question_type == 'Question about Content': + with st.spinner('Processing request... don\'t leave this page!'): + response, evidence, source = prompt_question_about_content_with_reranking(question, library_name, filename, topic) + + st.write('### Response') + st.write(response) + + if evidence: + st.write('### Evidence') + st.write('"' + evidence + '"') + + if source: + st.write('### File source') + st.write(source) diff --git a/solutions/use_cases/lecture_tool/pages/4_Summarizer.py b/solutions/use_cases/lecture_tool/pages/4_Summarizer.py new file mode 100644 index 0000000..882fb3f --- /dev/null +++ b/solutions/use_cases/lecture_tool/pages/4_Summarizer.py @@ -0,0 +1,105 @@ +from llmware.library import Library +from llmware.prompts import Prompt, Sources +from llmware.retrieval import Query + +import streamlit as st + +import os +import sys + +sys.path.insert(0, os.getcwd()) + +from Utils import get_stored_files, get_stored_libraries + + +ACCOUNT_NAME = 'lecture_tool' +SUMMARIZER_MODEL = 'slim-summary-tool' + + +# +# Summarizes specified file in specified library. +# +# Performs a text query for the topic, which can be an empty string. +# +# Uses the SUMMARIZER_MODEL defined above to generate a summary. +# +@st.cache_data(show_spinner=False) +def summarize_file(library_name, filename, topic): + # Load in appropriate library + library = Library().load_library(library_name, account_name=ACCOUNT_NAME) + print('\nupdate: library card - ', library.get_library_card()) + + # Create Query object + query = Query(library) + + # Load in appropriate model + summarizer_prompter = Prompt().load_model(SUMMARIZER_MODEL, temperature=0.0, sample=False) + + # Access all text blocks corresponding to specified file if no topic is provided + if topic == '': + # Filter out text chunks by filename + print('\nupdate: no topic provided, summarizing entire library') + query_results = query.apply_custom_filter(query.get_whole_library(), {'file_source': filename}) + + # Change key in query results for compatibility with RAG call + for result in query_results: + result['text'] = result['text_search'] + del result['text_search'] + # Access the text blocks corresponding to the specified file and topic + else: + # Perform text query for the topic, then filter out text blocks by filename + print('\nupdate: topic provided, performing text query') + query_results = query.apply_custom_filter(query.text_query(topic), {'file_source': filename}) + + print('\nupdate: correct library chunks - ', query_results) + + # Pass in appropriate text blocks as source to the model + sources = Sources(summarizer_prompter).package_source(query_results, aggregate_source=True) + print('\nupdate: sources - ', sources) + + # Prompt the model for a summary + print('\nupdate: summarizing in process') + response = summarizer_prompter.prompt_with_source('key points', first_source_only=False, verbose=True) + print('\nupdate: response - ', response) + + # Create a list of only the unique points generated by the model + key_points = [] + for resp in response: + for point in resp["llm_response"]: + if point not in key_points: + if point.strip(): + if not point.strip().startswith("Not Found"): + key_points.append('- ' + point) + + return key_points + + +# +# Main block for GUI logic. +# +if __name__ == '__main__': + st.title('Summarize your lectures') + + st.write('### Prompt info') + + library_name = st.selectbox( + 'Select the library:', + tuple(get_stored_libraries()) + ) + + if library_name: + filename = st.selectbox( + 'Select the file:', + tuple(get_stored_files(library_name)) + ) + + topic = st.text_input('Optionally enter a topic to summarize:') + + if (st.button('Summarize')): + with st.spinner('Summarizing transcript... don\'t leave this page!'): + response = summarize_file(library_name, filename, topic) + + st.write('### Summary') + + for point in response: + st.write(point) diff --git a/solutions/use_cases/lecture_tool/pages/5_Transcripts.py b/solutions/use_cases/lecture_tool/pages/5_Transcripts.py new file mode 100644 index 0000000..7ba28ad --- /dev/null +++ b/solutions/use_cases/lecture_tool/pages/5_Transcripts.py @@ -0,0 +1,59 @@ +from llmware.library import Library +from llmware.retrieval import Query + +import streamlit as st + +import os +import sys + +sys.path.insert(0, os.getcwd()) + +from Utils import get_stored_files, get_stored_libraries + +ACCOUNT_NAME = 'lecture_tool' + + +# +# Accesses transcription by concatenating text from each block corresponding to +# the specified filename in the specified library. +# +@st.cache_data(show_spinner=False) +def get_transcript(library_name, filename): + library = Library().load_library(library_name, account_name=ACCOUNT_NAME) + print('\nupdate: library card - ', library.get_library_card()) + + query = Query(library) + + text = '' + for result in query.get_whole_library(): + if result['file_source'] == filename: + text += result['text_search'] + + return text + + +# +# Main block for GUI logic. +# +if __name__ == '__main__': + st.title('View your transcripts') + + st.write('### Prompt info') + + library_name = st.selectbox( + 'Select the library:', + tuple(get_stored_libraries()) + ) + + if library_name: + filename = st.selectbox( + 'Select the file:', + tuple(get_stored_files(library_name)) + ) + + if (st.button('View')): + with st.spinner('Loading transcript... don\'t leave this page!'): + response = get_transcript(library_name, filename) + + st.write('### Transcript') + st.write(response) diff --git a/solutions/use_cases/lecture_tool/requirements.txt b/solutions/use_cases/lecture_tool/requirements.txt new file mode 100644 index 0000000..be8bf32 --- /dev/null +++ b/solutions/use_cases/lecture_tool/requirements.txt @@ -0,0 +1,2 @@ +llmware +streamlit \ No newline at end of file diff --git a/solutions/use_cases/lecture_tool/sample_audio_files/01. Course Introduction Rome39;s Greatness and First Crises-YTConverter.app.wav b/solutions/use_cases/lecture_tool/sample_audio_files/01. Course Introduction Rome39;s Greatness and First Crises-YTConverter.app.wav new file mode 100644 index 0000000..6ccf3c5 Binary files /dev/null and b/solutions/use_cases/lecture_tool/sample_audio_files/01. Course Introduction Rome39;s Greatness and First Crises-YTConverter.app.wav differ diff --git a/solutions/use_cases/lecture_tool/sample_audio_files/1. Introduction, Course Organization of MIT 7.016 Introductory Biology, Fall 2018-YTConverter.app.mp3 b/solutions/use_cases/lecture_tool/sample_audio_files/1. Introduction, Course Organization of MIT 7.016 Introductory Biology, Fall 2018-YTConverter.app.mp3 new file mode 100644 index 0000000..70bfc84 Binary files /dev/null and b/solutions/use_cases/lecture_tool/sample_audio_files/1. Introduction, Course Organization of MIT 7.016 Introductory Biology, Fall 2018-YTConverter.app.mp3 differ diff --git a/solutions/use_cases/lecture_tool/sample_audio_files/Information Flow - Biol 112 at UBC.mp3 b/solutions/use_cases/lecture_tool/sample_audio_files/Information Flow - Biol 112 at UBC.mp3 new file mode 100644 index 0000000..b3a2a43 Binary files /dev/null and b/solutions/use_cases/lecture_tool/sample_audio_files/Information Flow - Biol 112 at UBC.mp3 differ diff --git a/solutions/use_cases/lecture_tool/sample_audio_files/MIT Introduction to Python.wav b/solutions/use_cases/lecture_tool/sample_audio_files/MIT Introduction to Python.wav new file mode 100644 index 0000000..babd846 Binary files /dev/null and b/solutions/use_cases/lecture_tool/sample_audio_files/MIT Introduction to Python.wav differ diff --git a/solutions/use_cases/msa_processing.py b/solutions/use_cases/msa_processing.py new file mode 100644 index 0000000..9d373d5 --- /dev/null +++ b/solutions/use_cases/msa_processing.py @@ -0,0 +1,87 @@ + +""" This example shows an end-to-end processing of Master Services Agreements (MSAs) - including the parsing and +text chunking of the documents with document filtering to rapidly identify the "MSA" agreements from a large +batch of contract documents, using queries to extract source materials, using a locally-running GPU to review and +answer the key questions, with evidence checking, and output for final human review. + + The example uses a quantized 6B parameter model running on a local machine. + + Note: this example tracks the example #6 in the Fast Start. + +""" + +import os + +from llmware.setup import Setup +from llmware.library import Library +from llmware.prompts import Prompt, HumanInTheLoop +from llmware.retrieval import Query +from llmware.configs import LLMWareConfig + + +def msa_processing(): + + local_path = Setup().load_sample_files() + agreements_path = os.path.join(local_path, "AgreementsLarge") + + # create a library with all of the Agreements (~80 contracts) + msa_lib = Library().create_new_library("msa_lib503_635") + msa_lib.add_files(agreements_path) + + # find the "master service agreements" (MSA) + q = Query(msa_lib) + query = "master services agreement" + results = q.text_search_by_page(query, page_num=1, results_only=False) + + # results_only = False will return a dictionary with 4 keys: {"query", "results", "doc_ID", "file_source"} + msa_docs = results["file_source"] + + # load prompt/llm locally + model_name = "llmware/dragon-yi-6b-gguf" + prompter = Prompt().load_model(model_name) + + # analyze each MSA - "query" & "llm prompt" + for i, docs in enumerate(msa_docs): + + print("\n") + print (i+1, "Reviewing MSA - ", docs) + + # look for the termination provisions in each document + doc_filter = {"file_source": [docs]} + termination_provisions = q.text_query_with_document_filter("termination", doc_filter) + + # package the provisions as a source to a prompt + sources = prompter.add_source_query_results(termination_provisions) + + print("update: sources - ", sources) + + # call the LLM and ask our question + response = prompter.prompt_with_source("What is the notice for termination for convenience?") + + # post processing fact checking + stats = prompter.evidence_comparison_stats(response) + ev_source = prompter.evidence_check_sources(response) + + for i, resp in enumerate(response): + print("update: llm response - ", resp) + print("update: compare with evidence- ", stats[i]["comparison_stats"]) + print("update: sources - ", ev_source[i]["source_review"]) + + prompter.clear_source_materials() + + # Save jsonl report with full transaction history to /prompt_history folder + print("\nupdate: prompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + + prompter.save_state() + + # Generate CSV report for easy Human review in Excel + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + + print("\nupdate: csv output for human review - ", csv_output) + + return 0 + + +if __name__ == "__main__": + + m = msa_processing() diff --git a/solutions/use_cases/parsing_great_speeches.py b/solutions/use_cases/parsing_great_speeches.py new file mode 100644 index 0000000..e1c5475 --- /dev/null +++ b/solutions/use_cases/parsing_great_speeches.py @@ -0,0 +1,84 @@ + +""" This example illustrates how to parse voice-to-text files in memory, run searches against those files, and then +use to run extract inference calls to identify specific content - and in the end, produce a list of the key source +citations with source file, time_stamp, and target text. """ + + +import os + +from llmware.parsers import Parser +from llmware.setup import Setup +from llmware.util import Utilities +from llmware.models import ModelCatalog + + +def greatest_speeches_example(): + + # four sample file sets - "famous_quotes" | "greatest_speeches" | "youtube_demos" | "earnings_calls" + voice_sample_files = Setup().load_voice_sample_files(small_only=False) + + input_folder = os.path.join(voice_sample_files, "greatest_speeches") + + print("\nStep 1 - converting, parsing and text chunking ~56 WAV files") + + parser_output = Parser(chunk_size=400, max_chunk_size=600).parse_voice(input_folder, + write_to_db=False, + copy_to_library=False, + remove_segment_markers=True, + chunk_by_segment=True, + real_time_progress=False) + + print("\nStep 2- look at the text chunks with all of the metadata") + + for i, entries in enumerate(parser_output): + print("all parsed blocks: ", i, entries) + + print("\nStep 3- run an inline text search for 'president'") + + results = Utilities().fast_search_dicts("president", parser_output) + + for i, res in enumerate(results): + print("search results: ", i, res) + + print("\nStep 4- use LLM to review each search result - and if specific U.S. presidents found, then display the source") + + extract_model = ModelCatalog().load_model("slim-extract-tool", sample=False, temperature=0.0, max_output=200) + + final_list = [] + + for i, res in enumerate(results): + + response = extract_model.function_call(res["text"], params=["president name"]) + + various_american_presidents = ["kennedy", "carter", "nixon", "reagan", "clinton", "obama"] + + extracted_name = "" + if "president_name" in response["llm_response"]: + if len(response["llm_response"]["president_name"]) > 0: + extracted_name = response["llm_response"]["president_name"][0].lower() + else: + print("\nupdate: skipping result - no president name found - ", response["llm_response"], res["text"]) + + for president in various_american_presidents: + if president in extracted_name: + print("\nextracted american president text: ", i, extracted_name) + print("file source: ", res["file_source"]) + print("time stamp: ", res["coords_x"], res["coords_y"], res["coords_cx"], res["coords_cy"]) + print("text: ", i, res["text"]) + final_list.append({"key": president, "source": res["file_source"], "time_start": res["coords_x"], + "text": res["text"]}) + + print("\nStep 5 - final results") + for i, f in enumerate(final_list): + print("final results: ", i, f) + + return final_list + + +if __name__ == "__main__": + + greatest_speeches_example() + + + + diff --git a/solutions/use_cases/slicing_and_dicing_office_docs.py b/solutions/use_cases/slicing_and_dicing_office_docs.py new file mode 100644 index 0000000..9f31c89 --- /dev/null +++ b/solutions/use_cases/slicing_and_dicing_office_docs.py @@ -0,0 +1,117 @@ + +""" 'SLICING & DICING like a Data Ninja' - this example shows a variety of parsing, extraction, and packaging +techniques. + + This example builds on the 'Microsoft IR' file set, which is included in the llmware sample files. + + -- published on Microsoft IR website: https://www.microsoft.com/en-us/investor + -- zip archives of the investor kit for each of the quarters since 2020 + + We will parse the zipped files, which contain ~150 Powerpoints, Word and Excel files. + + Note: parsing ZIP files is easy as they are automatically unzipped, and the individual files are then + routed to the appropriate parser. + + After completing the parsing: + + 1. OCR the images - we will run an OCR against all of the extracted images and add that text into our + library collection. + + -NOTE: to complete this OCR step requires two additional dependencies be installed: + -- pytesseract - `pip3 install pytesseract` + -- libtesseract - `brew install tesseract` or `sudo apt install libtesseract-dev` or + download executable and install through GUI on Windows + + 2. Extract the tables to CSV - we will export all of the tables we found (which are indexed in the DB), + and publish out as individual .CSV files. + + 3. Extract all of the images - and take a look at where we can find the individual .PNG and .JPEG files. + + 4. Dump the whole database into a JSONL file. + + 5. Create and configure a 'model-ready' dataset. + +""" + + +from llmware.library import Library +from llmware.configs import LLMWareConfig +from llmware.setup import Setup +from llmware.retrieval import Query +from llmware.dataset_tools import Datasets + + +def create_microsoft_ir_library(library_name="microsoft_ir_350"): + + """ Pulls the Microsoft IR sample files - parses, text chunks and creates Library. """ + + print("update: downloading and caching microsoft investor relations sample files") + + microsoft_ir = Setup().load_selected_sample_files(sample_folder="microsoft_ir") + + print("update: completed downloading files @ files path: ", microsoft_ir) + + my_lib = Library().create_new_library(library_name) + + # pass the zip archives like any other file in .add_files method + parsing_output = my_lib.add_files(microsoft_ir, chunk_size=400, max_chunk_size=600,smart_chunking=1, + get_tables=True, get_images=True) + + print("update: parsing output: ", parsing_output) + + # check out the images extracted + print("update: images extracted to path: ", my_lib.image_path) + + # run a quick query + qr = Query(my_lib).text_query("azure", result_count=10) + + for i, res in enumerate(qr): + print("results: ", i, res) + + return 0 + + +def slice_and_dice_special(ln): + + # Step 1 - load a library that we have created + lib = Library().load_library(ln) + + # Step 2 - export all of the tables to CSV + q = Query(lib) + extracted_tables = q.export_all_tables(output_fp=lib.output_path) + + print("extracted tables summary: ", extracted_tables) + + # Step 3 - run OCR on all of the extracted images + # to run OCR: need to install pytesseract & lib tesseract + + lib.run_ocr_on_images(add_to_library=True, chunk_size=400, min_size=10, realtime_progress=True) + print("done with ocr processing") + + # Step 4 - export the whole library to jsonl file + output = lib.export_library_to_jsonl_file(lib.output_path, "microsoft_ir_lib") + + # Step 5 - create dataset + ds = Datasets(library=lib, testing_split=0.10, validation_split=0.10, ds_id_mode="random_number") + ds_output = ds.build_text_ds(min_tokens=100, max_tokens=500) + + print("done with dataset") + + return True + + +# main execution script starts here + +if __name__ == "__main__": + + # select collection db - mongo, sqlite, or postgres + LLMWareConfig().set_active_db("sqlite") + + ln = "microsoft_investor_relations_350" + + # build the library (only required once) + create_microsoft_ir_library(library_name=ln) + + # run the slicing and dicing + slice_and_dice_special(ln) + diff --git a/solutions/use_cases/using-llm-for-table-reading.py b/solutions/use_cases/using-llm-for-table-reading.py new file mode 100644 index 0000000..e1b7e28 --- /dev/null +++ b/solutions/use_cases/using-llm-for-table-reading.py @@ -0,0 +1,232 @@ + +""" USING-LLM-FOR-TABLE-READING Recipes - this example consists of 3 recipes that illustrate the building blocks +of using locally-deployed small specialized language models for table question-answering with complex financial +and business documents. + + Note: this is a *** leading-edge *** set of recipes - it won't always work perfectly out of the box, and generally + will require some tinkering with the pre-processing and post-processing and strategies at each step to + improve accuracy. + + LLMs used: + -- table reading: + -- dragon-qwen2-7b-gguf + -- dragon-yi-9b-gguf + + -- semantic reranker: jina-reranker-turbo (example 3) + + The recipes build on each other: + + Example 1 - basic recipe for using a LLM to answer a question based on a table text passage + + Example 2 - integrates parsing - parses a 10K document - extracts key tables - and then asks questions + directly against the parsed, extracted tables (assumes we know the right questions to ask + each table. + + Example 3 - integrates semantic similarity - parses and extracts tables from 10K, applies a semantic reranker + to identify the table with the highest semantic similarity to our question, and then 'chooses' + that table, and then runs the inference with the question and the highest ranked table. + """ + +import os +import re +from llmware.models import ModelCatalog +from llmware.parsers import Parser +from llmware.setup import Setup +from llmware.util import Utilities + + +def example1_getting_started (model_name="dragon-qwen-7b-gguf"): + + """ Basic recipe for running an inference to read a table. """ + + # sample table text with \t separators between items + + text = ("\t2022 12/31/22\t2021 12/31/21\t2020 12/31/20\t2019 12/31/19 NET SALES OR REVENUES" + "\t81,462\t53,823\t31,536\t24,578 Cost of Goods Sold (Excl Depreciation)\t57,066\t37,306\t22,584\t18,402 " + "Depreciation, Depletion And Amortization\t3,543\t2,911\t2,322\t2,107 Depreciation\t2,655\t2,146\t1,802\t1,298 " + "Amortization of Intangibles\t888\t51\t51\t44Amortization of Deferred Charges\t--\t714\t469\t765 GROSS " + "INCOME\t20,853\t13,606\t6,630\t4,069 Selling, General & Admin Expenses\t7,021\t7,110\t4,636\t3,989 " + "Research and Development Expense\t3,075\t2,593\t1,491\t1,343OPERATING INCOME\t13,832\t6,496\t1,994\t80 " + "Extraordinary Charge - Pretax\t(228)\t(101)\t0\t(196) Non-Operating Interest Income\t297\t56\t30\t44 " + "Other Income/Expenses - Net\t(15)\t263\t(122)\t92 Interest Expense On Debt\t167\t424\t796\t716 Interest " + "Capitalized\t0\t53\t48\t31 PRETAX INCOME\t13,719\t6,343\t1,154\t(665) Income " + "Taxes\t(1,132)\t(699)\t(292)\t(110) Current Domestic Income Tax\t62\t9\t4\t5 Current Foreign Income " + "Tax\t1,266\t839\t248\t86Deferred Domestic Income Tax\t27\t0\t0\t(4) Deferred Foreign Income " + "Tax\t(223)\t(149)\t40\t23 Minority Interest\t4\t120\t172\t95 NET INCOME BEFORE EXTRA ITEMS/PREFERRED " + "DIVIDENDS\t12,583\t5,524\t690\t(870) NET INCOME USED TO CALCULATE BASIC EARNINGS PER " + "SHARE\t12,583\t5,524\t690\t(870) Shares used in computing earnings per share - " + "Fully Diluted\t3,475\t3,387\t3,249\t2,661Earning per Common Share - " + "Basic\t4.02\t1.87\t0.25\t(0.33)Earning per Common Share - Fully Diluted\t3.62\t1.63\t0.21\t(0.33)() = " + "Negative Values In U.S. Dollars Values are displayed in Millions except for earnings per share " + "and where noted") + + questions = ["What is the pretax income in 2022?", + "What is the pretax income in 2021?", + "What is the amount of depreciation in 2020?", + "What is the fully diluted earnings per share in 2022?", + "What is the amount of dividends in 2021 and 2022?", + "What is the SG&A expense in 2022?", + "What were revenues in 2019?"] + + model = ModelCatalog().load_model(model_name, temperature=0.0, sample=False) + + for question in questions: + print("question: ", question) + response = model.inference(question, add_context=text) + print("response: ", response) + + return True + + +def example2_parse_tables_and_ask_questions(model_name="dragon-qwen-7b-gguf"): + + """ Parse a 10K, extract key tables, and then ask specific questions to specific tables. """ + + sample_files = Setup().load_sample_files() + folder = "FinDocs" + fp = os.path.join(sample_files, folder) + fn = "Amazon-2021-Annual-Report.pdf" + + # table_grid = True will provide a HTML representation of the table + # table_grid = False will provide a simpler /t and /n separators in representing the table + + parser_output = Parser(table_grid=False).parse_one_pdf(fp,fn) + + tables = [] + for i, chunks in enumerate(parser_output): + if chunks["content_type"] == "table": + print("text chunks: ", i, chunks) + tables.append(chunks["table"]) + + questions_by_table = [ + ["What is the amount of owned square footage of office space in North America?", + "How many international stores?" + ], + + ["What was the amount of cash at the end of 2020?", + "What was the amount of cash at the end of 2021?", + "What was net income in 2020?", + "What was stock compensation in 2021?", + "What was the amount of net increase in cash in 2020?" + ], + + ["What were total net sales in 2021?"], + + ["What is the balance amount on January 1, 2019?"] + + ] + + model = ModelCatalog().load_model(model_name, temperature=0.0, sample=False) + + for t, table in enumerate(tables): + + print("\nEvaluating table: ", t, table) + + for q, question in enumerate(questions_by_table[t]): + + print("\nQuestion: ", q, question) + response = model.inference(question, add_context=table) + print("answer: ", response) + + return True + + +def example3_table_reading_e2e(model_name="dragon-qwen-7b-gguf"): + + """ Will parse, extract tables, apply semantic reranking to find the best fit table, and then ask the + key question only to that table. """ + + """ Parse a 10K, extract key tables, and then ask specific questions to specific tables. """ + + question = "What was the amount of cash at the end of 2020?" + + # Step 1 - parse and extract the tables from 10K + + sample_files = Setup().load_sample_files() + folder = "FinDocs" + fp = os.path.join(sample_files, folder) + fn = "Amazon-2021-Annual-Report.pdf" + + # table_grid = True will provide a HTML representation of the table + # table_grid = False will provide a simpler /t and /n separators in representing the table + + parser_output = Parser(table_grid=False).parse_one_pdf(fp, fn) + + tables = [] + print(f"\nStep 1 - parsing output - {fn} - created {len(parser_output)} text chunks total.") + + for i, chunks in enumerate(parser_output): + if chunks["content_type"] == "table": + print("tables found: ", i, chunks) + + if len(chunks["table"]) > 100: + text_snippet = str(chunks["table"][0:100]) + else: + text_snippet = chunks["table"] + + # optional / clean up the text snippet for display on screen + text_snippet = re.sub(r"[\n\r\t]","", text_snippet) + + tables.append({"text":chunks["table"], + "page_num": chunks["master_index"], + "file_source": chunks["file_source"], + "text_snippet": text_snippet}) + + # Step 2 - apply semantic ranking to compare the question with the extracted tables to find the + # table most likely to provide the answer + + print(f"\nStep 2 - select the extracted table most likely to be able to answer the question.") + + print("--option 1 - simple text search option (illustrative)") + exact_key = "CASH EQUIVALENTS" + results = Utilities().fast_search_dicts(exact_key,tables) + for i, res in enumerate(results): + print("text search results: ", i, exact_key,res) + + # could use as a substitute below + top_result = results[0] + + print(f"\n--option 2 - semantic similarity ranking (selected method)") + + reranker_model = ModelCatalog().load_model("jina-reranker-turbo") + output = reranker_model.inference(question, tables) + + for i, ranking in enumerate(output): + + if i==0: + print("TOP TABLE - ", i, ranking["rerank_score"], ranking["text_snippet"]) + else: + print(i, ranking["rerank_score"], ranking["text_snippet"]) + + top_table = output[0] + + print("\nTOP TABLE SOURCE: ", top_table["text"]) + + # Step 3 - run the query against the table to get the answer + + print(f"\nStep 3 - loading dragon model to answer the question using the 'top table' found") + + model = ModelCatalog().load_model(model_name, temperature=0.0, sample=False) + + print("question: ", question) + response = model.inference(question, add_context=top_table["text"]) + print("response: ", response) + print("source: ", top_table["file_source"]) + print("page num: ", top_table["page_num"]) + + return response + + +if __name__ == "__main__": + + # we would recommend either "dragon-qwen-7b-gguf" or "dragon-yi-9b-gguf" + + # shows basic recipe for passing a table context and asking a question + example1_getting_started(model_name="dragon-qwen-7b-gguf") + + # parse, extract table, ask questions to tables + example2_parse_tables_and_ask_questions(model_name="dragon-qwen-7b-gguf") + + # *leading edge* - ask a question to a 100 page pdf 10k, find the right table, and get answer from it + example3_table_reading_e2e(model_name="dragon-qwen-7b-gguf") + diff --git a/solutions/use_cases/web_services_slim_fx.py b/solutions/use_cases/web_services_slim_fx.py new file mode 100644 index 0000000..52abd44 --- /dev/null +++ b/solutions/use_cases/web_services_slim_fx.py @@ -0,0 +1,203 @@ + +""" This example illustrates a more complex recipe to generate a structured financial research dictionary with + 30 keys and values produced, using a combination of models and web services: + + Models + 1. slim-extract-tool + 2. slim-summary-tool + 3. bling-stablelm-3b-tool + + Web Services + 1. Yfinance - stock ticker + 2. Wikipedia - company background information + + The example shows how to extract keys from one source that can then be used as a lookup in a web service to + supplement the original source materials, and provide a secondary source, which can then also be prompted and + used to extract, analyze and summarize key information. + + NOTE: to run this example, please install yfinance library, e.g., 'pip3 install yfinance' + + """ + + +from llmware.web_services import YFinance +from llmware.models import ModelCatalog +from llmware.parsers import WikiParser + +from importlib import util +if not util.find_spec("yfinance"): + print("\nto run this example, you need to install yfinance first, e.g., pip3 install yfinance") + +# our input - financial news article + +text=("BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its " + "third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter " + "of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* " + "at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis " + "NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis " + "Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin " + "increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges " + "Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring " + "charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these " + "charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to " + "drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by " + "the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and " + "work with our wholesale partners to elevate and grow the marketplace.") + + +def research_example1(): + + """ End-to-end example generating 30 output key:value pairs """ + + # load three models in this example + + model = ModelCatalog().load_model("slim-extract-tool", temperature=0.0, sample=False) + model2 = ModelCatalog().load_model("slim-summary-tool", sample=False,temperature=0.0,max_output=200) + model3 = ModelCatalog().load_model("bling-stablelm-3b-tool", sample=False, temperature=0.0) + + research_summary = {} + + # extract information from the source materials + + extract_keys = ["stock ticker", "company name", + "total revenues", "restructuring charges", + "digital growth", "ceo comment", "quarter end date"] + + print("\nStep 1 - extract information from source text\n") + + for keys in extract_keys: + response = model.function_call(text,params=[keys]) + dict_keys = keys.replace(" ", "_") + print(f"update: extracting - {keys} - {response['llm_response']}") + if dict_keys in response["llm_response"]: + value = response["llm_response"][dict_keys][0] + research_summary.update({dict_keys: value}) + else: + print("could not find look up key successfully - ", response["llm_response"]) + + # secondary lookups using extracted information + + print("\nStep 2 - use extracted stock ticker in web service lookup to YFinance\n") + + if "stock_ticker" in research_summary: + ticker = research_summary["stock_ticker"] + # a little kludge related to yfinance api + ticker_core = ticker.split(":")[-1] + + yf = YFinance().get_stock_summary(ticker=ticker_core) + print("yahoo finance stock info: ", yf) + + research_summary.update({"current_stock_price": yf["currentPrice"]}) + research_summary.update({"high_ltm": yf["fiftyTwoWeekHigh"]}) + research_summary.update({"low_ltm": yf["fiftyTwoWeekLow"]}) + research_summary.update({"trailing_pe": yf["trailingPE"]}) + research_summary.update({"forward_pe": yf["forwardPE"]}) + research_summary.update({"volume": yf["volume"]}) + + yf2 = YFinance().get_financial_summary(ticker=ticker_core) + print("yahoo finance fin info - ", yf2) + research_summary.update({"market_cap": yf2["marketCap"]}) + research_summary.update({"price_to_sales": yf2["priceToSalesTrailing12Months"]}) + research_summary.update({"revenue_growth": yf2["revenueGrowth"]}) + research_summary.update({"ebitda": yf2["ebitda"]}) + research_summary.update({"gross_margin": yf2["grossMargins"]}) + research_summary.update({"currency": yf2["currency"]}) + + yf3 = YFinance().get_company_summary(ticker=ticker_core) + print("yahoo finance company info - ", yf3) + research_summary.update({"sector": yf3["sector"]}) + research_summary.update({"website": yf3["website"]}) + research_summary.update({"industry": yf3["industry"]}) + research_summary.update({"employees": yf3["fullTimeEmployees"]}) + + execs = [] + if "companyOfficers" in yf3: + for entries in yf3["companyOfficers"]: + if "totalPay" in entries: + pay = entries["totalPay"] + else: + pay = "pay-NA" + + if "age" in entries: + age = entries["age"] + else: + age = "age-NA" + + execs.append((entries["name"], entries["title"], age, pay)) + research_summary.update({"officers": execs}) + + print("\nStep 3 - use extracted company name to lookup in Wikipedia web service - and add background data\n") + + if "company_name" in research_summary: + + company_name = research_summary["company_name"] + output = WikiParser().add_wiki_topic(company_name, target_results=1) + + # get company summary + company_overview = "" + for i, blocks in enumerate(output["blocks"]): + if i < 3: + company_overview += blocks["text"] + + # call summary model to summarize + print("-- calling summary model to summarize the first part of the Wikipedia article") + summary = model2.function_call(company_overview, params=["company history (5)"]) + print("-- slim-summary - summary (5 points): ", summary) + + research_summary.update({"summary": summary["llm_response"]}) + + # get founding date + print("\n-- calling extract model to get key piece of information from the Wikipedia article - company founding date") + response = model.function_call(company_overview, params=["founding date"]) + print("-- slim-extract - founding date: ", response) + + research_summary.update({"founding_date": response["llm_response"]["founding_date"][0]}) + + print("\n-- calling extract model to get a short company business") + response = model.function_call(company_overview, params=["company description"]) + print("-- slim-extract - response: ", response) + research_summary.update({"company_description": response["llm_response"]["company_description"][0]}) + + # ask other questions directly + print("\n-- asking a question directly to the Wikipedia article about the business") + response = model3.inference("What is an overview of company's business?", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"business_overview": response["llm_response"] }) + + print("\n-- asking a question about the origin of the company's name") + response = model3.inference("What is the origin of the company's name?", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"origin_of_name": response["llm_response"]}) + + print("\n-- asking a question about the company's products") + response = model3.inference("What are the product names", add_context=company_overview) + print("-- bling-answer model - response: ", response) + research_summary.update({"products": response["llm_response"]}) + + print("\n\nStep 4 - Completed Research - Summary Output\n") + print("research summary: ", research_summary) + + item_counter = 1 + + for keys, values in research_summary.items(): + if isinstance(values, str): + + values = values.replace("\n", "") + values = values.replace("\r", "") + values = values.replace("\t", "") + + print(f"\t\t -- {item_counter} - \t - {keys.ljust(25)} - {str(values).ljust(40)}") + item_counter += 1 + + return research_summary + + +if __name__ == "__main__": + + research_example1() + + + + + + diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..5f37fa6 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,63 @@ +# Testing + +## Prereqs + +The tests rely on pytest and tabulate + +```bash +pip install pytest tabulate +``` + +To run tests specific to a particular database or vector database requires that the resources are up and running. + +Some tests may require API Keys to particular cloud model providers. If you would like to test that provider, then +you can either edit the test code to include your own, or update ```~/set-env.sh``` and run the following to get +the environment variables loaded in your shell: + +```bash +source ~/set-env.sh +``` + +### Running all tests + +```bash +python3 ./run-tests.py +``` + +Note: We do not currently recommend running all tests concurrently - we are updating our automation and test scope +to improve this capability going forward. + +WARNING: Initiating this script will "clean" the environment before running *_all tests_*: +- Uninstall any llmware modules locally installed +- Install the llmware module from this repo (simulating a user installing from pypi) +- Completely remove $HOME/llmware_data +- Reset Mongo and Milvus (drop all collections) -> not currently updated to drop Postgres and SQLITE DBs or other Vector DBs +- Run ./models/test_all_generative_models.py which will pull down many GBs of model files to the test systems. + + +### Only running tests from a specific folder or file + +This is our recommended testing approach, unless a PR has significant cross-module changes, and even in such cases +it is usually more effective to run 2-3 individual tests and supplement with manual testing. + +Note: please note that we are in the process of updating our test automation coverage suite. We would welcome +contributions from the community to continue to improve the automation and coverage scope of these tests. + +In addition to the formal tests, we would recommend using recent examples as well as good proxies, especially in +targeted areas of the code base. + +```bash +python3 ./run-tests.py library +``` + +```bash +python3 ./run-tests.py models/test_all_generative_models.py +``` + +### Including printed output while test is running + +By default pytest only shows output if tests fail. If you want see printed output for all tests add the _-s_ switch + +```bash +python3 ./run-tests.py library -s +``` diff --git a/tests/configs/test_account_overrides.py b/tests/configs/test_account_overrides.py new file mode 100644 index 0000000..9ee7721 --- /dev/null +++ b/tests/configs/test_account_overrides.py @@ -0,0 +1,40 @@ +""" Tests for custom account configuration over-rides to set up additional accounts, besides the default 'llmware'. + + By default, the test will run on MongoDB - to change the database (including no-install) - add: + + `LLMWareConfig().set_active_db("sqlite") + + """ + +import os + +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def test_setup_custom_account(): + + # creation of custom accounts -> 'implicit' creation allowed in llmware (no permission checks) + # assumed that upstream application will manage permission scope and account management + + account_name = "custom_account123" + library_name = "custom_lib123" + + # Create library and add files + library = Library().create_new_library(library_name, account_name=account_name) + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + + # Ensure the library's path was created with the right account name + assert account_name in library.library_main_path + + # additional check - build knowledge graph - confirm that artifacts in /nlp folder in custom account path + library.generate_knowledge_graph() + assert len(os.listdir(library.nlp_path)) > 0 + + # Do a query + results = Query(library).text_query("salary") + assert len(results) > 0 + diff --git a/tests/configs/test_path_overrides.py b/tests/configs/test_path_overrides.py new file mode 100644 index 0000000..2f60f92 --- /dev/null +++ b/tests/configs/test_path_overrides.py @@ -0,0 +1,64 @@ +""" Tests paths both for a standard account setup and custom account setup. + + Note: by default, this test will run on MongoDB - to switch DB used - add one-line: + + `LLMWareConfig().set_active_db("sqlite") + +""" + +import os + +from llmware.configs import LLMWareConfig +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup + + +def test_standard_account_setup(): + + library_name = "test_standard_account_setup" + library = Library().create_new_library(library_name) + + # add files + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + + # run test query + results = Query(library).text_query("base salary") + + assert len(results) > 0, "Expected results to be greater than 0 for standard account setup." + + +# Test overriding llmware_data folder +def test_setup_custom_location(): + + # upstream application creating custom home path, e.g., aibloks + new_home_folder = "llmware_data_custom" + LLMWareConfig.set_home(os.path.join(LLMWareConfig.get_home(), new_home_folder)) + + new_llmware_path = "aibloks_data" + LLMWareConfig.set_llmware_path_name(new_llmware_path) + + # Make folders if they don't already exist + if not os.path.exists(LLMWareConfig().get_home()): + os.mkdir(LLMWareConfig().get_home()) + if not os.path.exists(LLMWareConfig().get_llmware_path()): + LLMWareConfig.setup_llmware_workspace() + + assert new_home_folder in LLMWareConfig.get_home(), "Custom home folder not set correctly." + assert new_home_folder in LLMWareConfig.get_llmware_path() and new_llmware_path in LLMWareConfig.get_llmware_path(), "Custom LLMWare path not set correctly." + assert new_home_folder in LLMWareConfig.get_library_path() and new_llmware_path in LLMWareConfig.get_library_path(), "Library path not set correctly." + + # test with library setup and adding files + library_name = "library_with_custom_path" + library = Library().create_new_library(library_name) + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"Agreements")) + + assert new_home_folder in library.library_main_path, "Custom path not found in library main path." + + results = Query(library).text_query("salary") + + assert len(results) > 0 + + diff --git a/tests/embeddings/test_all_embedding_dbs.py b/tests/embeddings/test_all_embedding_dbs.py new file mode 100644 index 0000000..b3fe22e --- /dev/null +++ b/tests/embeddings/test_all_embedding_dbs.py @@ -0,0 +1,120 @@ + +""" Provides a set of short tests for specific vector DB. """ + +import os +import pytest +import time + +from llmware.library import Library +from llmware.embeddings import EmbeddingHandler +from llmware.exceptions import UnsupportedEmbeddingDatabaseException +from llmware.models import ModelCatalog +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.configs import LLMWareConfig +from llmware.resources import CloudBucketManager +from tests.embeddings.utils import qdrant_installed + + +def test_unsupported_embedding_db(): + + embedding_db = "milvusXYZ" # Bad Embedding DB Name + with pytest.raises(UnsupportedEmbeddingDatabaseException) as excinfo: + embedding_handler = EmbeddingHandler(library=None) + embedding_summary = embedding_handler.create_new_embedding(embedding_db=embedding_db, model=None) + assert str(excinfo.value) == f"'{embedding_db}' is not a supported vector embedding database" + + +def test_milvus_embedding_and_query(): + + sample_files_path = Setup().load_sample_files() + library = Library().create_new_library("test_embedding_milvus") + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + results = generic_embedding_and_query(library, "milvus") + assert len(results) > 0 + library.delete_library(confirm_delete=True) + + +def test_neo4j_embedding_and_query(): + + sample_files_path = Setup().load_sample_files() + library = Library().create_new_library("test_embedding_neo4j") + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + results = generic_embedding_and_query(library, "neo4j") + assert len(results) > 0 + library.delete_library(confirm_delete=True) + + +def test_chromadb_embedding_and_query(): + + sample_files_path = Setup().load_sample_files() + library = Library().create_new_library("test_embedding_neo4j") + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + results = generic_embedding_and_query(library, "chromadb") + assert len(results) > 0 + library.delete_library(confirm_delete=True) + + +def test_faiss_embedding_and_query(): + + sample_files_path = Setup().load_sample_files() + library = Library().create_new_library("test_embedding_faiss") + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + results = generic_embedding_and_query(library, "faiss") + assert len(results) > 0 + library.delete_library(confirm_delete=True) + + +def test_lancedb_embedding_and_query(): + + sample_files_path = Setup().load_sample_files() + library = Library().create_new_library("test_embedding_lancedb") + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + results = generic_embedding_and_query(library, "lancedb") + assert len(results) > 0 + library.delete_library(confirm_delete=True) + + +@pytest.mark.skipif(not qdrant_installed(), reason="Qdrant client is not installed") +def test_qdrant_embedding_and_query(): + os.environ["USER_MANAGED_QDRANT_LOCATION"] = ":memory:" + sample_files_path = Setup().load_sample_files() + library = Library().create_new_library("test_embedding_qdrant") + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + results = generic_embedding_and_query(library, "qdrant") + assert len(results) > 0 + library.delete_library(confirm_delete=True) + +# def test_pinecone_embedding_and_query(): +# with pytest.raises(ImportError) as excinfo: +# library = None +# results = generic_embedding_and_query(library, "pinecone") +# assert 'pip install pinecone-client' in str(excinfo.value) + +# def test_pinecone_embedding_and_query(): +# sample_files_path = Setup().load_sample_files() +# library = Library().create_new_library("test_embedding") +# library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + +# os.environ["PINECONE_API_KEY"] = "abeab29e-7a48-426b-b17a-c74567720876" +# os.environ["PINECONE_ENVIRONMENT"] = "gcp-starter" + +# results = generic_embedding_and_query(library, "pinecone") +# assert len(results) > 0 +# library.delete_library(confirm_delete=True) + + +def generic_embedding_and_query(library, embedding_db): + + # Run the embeddings (only of first 3 docs ) + model=ModelCatalog().load_model("mini-lm-sbert") + embedding_handler = EmbeddingHandler(library=library) + embedding_summary = embedding_handler.create_new_embedding(embedding_db=embedding_db, model=model,doc_ids=[1, 2, 3, 4, 5]) + + # Do a query + query = "pact" + query_results = Query(library).semantic_query(query, result_count=5) + + # Delete the embedding + embedding_handler.delete_index(embedding_db=embedding_db,embedding_dims=embedding_summary["embedding_dims"], model_name=model.model_name) + return query_results diff --git a/tests/embeddings/test_embedding_model_load.py b/tests/embeddings/test_embedding_model_load.py new file mode 100644 index 0000000..1bce00c --- /dev/null +++ b/tests/embeddings/test_embedding_model_load.py @@ -0,0 +1,33 @@ + +""" Tests that embedding model is loaded and yielding a structurally correct embedding vector. """ + + +from llmware.models import ModelCatalog + + +def test_embedding_model_local_load(): + + emb_models = ModelCatalog().list_embedding_models() + + test_text = ("This is just a sample text to confirm that the embedding model is loading and correctly " + "converting into a structurally accurate embedding vector.") + + for model_card in emb_models: + + if model_card["model_family"] in ["HFEmbeddingModel"]: + + print(f"\nloading model - {model_card['model_name']} - embedding dims - {model_card['embedding_dims']}") + + model = ModelCatalog().load_model(model_card["model_name"]) + + embedding_vector = model.embedding(test_text) + + assert embedding_vector is not None + + print(f"created vector successfully with dimensions: ", embedding_vector[0].shape) + + assert embedding_vector[0].shape[0] == model_card['embedding_dims'] + + return 0 + + diff --git a/tests/embeddings/test_install_embeddings.py b/tests/embeddings/test_install_embeddings.py new file mode 100644 index 0000000..35054c4 --- /dev/null +++ b/tests/embeddings/test_install_embeddings.py @@ -0,0 +1,113 @@ + +""" Test for embedding vector creation and storage in a selected vector DB with selected embedding model. """ + + +import os +from llmware.library import Library +from llmware.retrieval import Query +from llmware.setup import Setup +from llmware.status import Status +from llmware.configs import LLMWareConfig + + +def setup_library(library_name): + + """ Note: this setup_library method is provided to enable a self-contained example to create a test library """ + + # Step 1 - Create library which is the main 'organizing construct' in llmware + print ("\nupdate: Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # check the embedding status 'before' installing the embedding + embedding_record = library.get_embedding_status() + print("embedding record - before embedding ", embedding_record) + + # Step 2 - Pull down the sample files from S3 through the .load_sample_files() command + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + + # Step 3 - point ".add_files" method to the folder of documents that was just created + # this method parses the documents, text chunks, and captures in database + + print("update: Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements"), + chunk_size=400, max_chunk_size=600, smart_chunking=1) + + return library + + +def test_install_vector_embeddings(): + + LLMWareConfig().set_active_db("sqlite") + + library = setup_library("test_emb_install_09123") + + # select vector db that you would like to test + vector_db = "chromadb" + + LLMWareConfig().set_vector_db(vector_db) + + # select embedding model + embedding_model = "mini-lm-sbert" + + library_name = library.library_name + + print(f"\nupdate: Starting the Embedding: " + f"library - {library_name} - " + f"vector_db - {vector_db} - " + f"model - {embedding_model}") + + # *** this is the one key line of code to create the embedding *** + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db,batch_size=100) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + # Start using the new vector embeddings with Query + sample_query = "incentive compensation" + print("\n\nupdate: Run a sample semantic/vector query: {}".format(sample_query)) + + # queries are constructed by creating a Query object, and passing a library as input + query_results = Query(library).semantic_query(sample_query, result_count=20) + + assert query_results is not None + + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # to see all of the dictionary keys returned, uncomment the line below + # print("update: query_results - all - ", i, entries) + + # for display purposes only, we will only show the first 125 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + # lets take a look at the library embedding status again at the end to confirm embeddings were created + embedding_record = library.get_embedding_status() + + assert embedding_record is not None + + print("\nupdate: embedding record - ", embedding_record) + + return 0 + + + + + diff --git a/tests/embeddings/test_sentence_transformers_load.py b/tests/embeddings/test_sentence_transformers_load.py new file mode 100644 index 0000000..f809f23 --- /dev/null +++ b/tests/embeddings/test_sentence_transformers_load.py @@ -0,0 +1,77 @@ + +""" Tests that sentence transformer model is loaded and yielding a structurally correct embedding vector. + +To use this test, you may need install the SentenceTransformer library as follows: + + -- pip3 install sentence-transformers + +""" + + +from llmware.models import ModelCatalog +from sentence_transformers import SentenceTransformer + + +def test_sentence_transformer_model_local_load(): + + # This model list was generated by here https://www.sbert.net/docs/pretrained_models.html and + # selecting the "All Models" switch + + sentence_transformer_models = [ + 'all-MiniLM-L12-v1', + 'all-MiniLM-L12-v2', + 'all-MiniLM-L6-v1', + 'all-MiniLM-L6-v2', + 'all-distilroberta-v1', + 'all-mpnet-base-v1', + 'all-mpnet-base-v2', + 'all-roberta-large-v1', + 'average_word_embeddings_glove.6B.300d', + 'average_word_embeddings_komninos', + 'gtr-t5-base', + 'gtr-t5-large', + 'gtr-t5-xl', + 'gtr-t5-xxl', + 'msmarco-bert-base-dot-v5', + 'msmarco-distilbert-base-tas-b', + 'msmarco-distilbert-dot-v5', + 'multi-qa-MiniLM-L6-cos-v1', + 'multi-qa-MiniLM-L6-dot-v1', + 'multi-qa-distilbert-cos-v1', + 'multi-qa-distilbert-dot-v1', + 'multi-qa-mpnet-base-cos-v1', + 'multi-qa-mpnet-base-dot-v1', + 'paraphrase-MiniLM-L12-v2', + 'paraphrase-MiniLM-L3-v2', + 'paraphrase-MiniLM-L6-v2', + 'paraphrase-TinyBERT-L6-v2', + 'paraphrase-albert-small-v2', + 'paraphrase-distilroberta-base-v2', + 'paraphrase-mpnet-base-v2', + 'paraphrase-multilingual-MiniLM-L12-v2', + 'paraphrase-multilingual-mpnet-base-v2', + 'sentence-t5-base', + 'sentence-t5-large', + 'sentence-t5-xl', + 'sentence-t5-xxl' + ] + + test_text = ("This is just a sample text to confirm that the embedding model is loading and correctly " + "converting into a structurally accurate embedding vector.") + + for model_name in sentence_transformer_models: + + print(f"\nloading sentence transformer model: {model_name}") + + st_model = SentenceTransformer(model_name) + model = ModelCatalog().load_sentence_transformer_model(st_model, model_name=model_name) + embedding_vector = model.embedding([test_text]) + + assert embedding_vector is not None + + print(f"created vector successfully with dimensions: ", embedding_vector.shape) + + return 0 + + +test_sentence_transformer_model_local_load() diff --git a/tests/library/test_library.py b/tests/library/test_library.py new file mode 100644 index 0000000..f15beeb --- /dev/null +++ b/tests/library/test_library.py @@ -0,0 +1,56 @@ + +""" Tests core library functions. By default, runs on MongoDB - to change: + + `LLMWareConfig().set_active_db("sqlite") + +""" + + +import os +import tempfile +from llmware.configs import LLMWareConfig +from llmware.library import Library, LibraryCatalog +from llmware.setup import Setup + + +def test_core_library_functions(): + + # Library creations + library_name = "LibraryABC" + library = Library().create_new_library(library_name) + + # Get all library names: + all_library_names = [] + for library_card in LibraryCatalog().all_library_cards(): + all_library_names.append(library_card["library_name"]) + + # Ensure library name exists + assert library_name in all_library_names + + # Add a few files + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + + # Library Export + temp_export_dir = tempfile.gettempdir() + json_export_path = library.export_library_to_jsonl_file(temp_export_dir, "lib_export") + assert os.path.getsize(json_export_path) > 100 + os.remove(json_export_path) + + # dump whole parsed library with only text + text_export_path = library.export_library_to_txt_file(temp_export_dir, "lib_export") + assert os.path.getsize(text_export_path) > 100 + os.remove(text_export_path) + + # Delete the library + library.delete_library(confirm_delete=True) + + # Get all library names: + all_library_names = [] + for library_card in LibraryCatalog().all_library_cards(): + all_library_names.append(library_card["library_name"]) + + # Ensure library name does not exist + assert library_name not in all_library_names + + diff --git a/tests/models/test_agent_llmfx_process.py b/tests/models/test_agent_llmfx_process.py new file mode 100644 index 0000000..bf71996 --- /dev/null +++ b/tests/models/test_agent_llmfx_process.py @@ -0,0 +1,66 @@ +"""Tests the execution of a multi-step Agent process using multiple SLIM models.""" + +from llmware.agents import LLMfx + +def test_multistep_agent_process(): + # Sample customer transcript + customer_transcript = "My name is Michael Jones, and I am a long-time customer. The Mixco product is not working currently, and it is having a negative impact on my business, as we can not deliver our products while it is down. This is the fourth time that I have called. My account number is 93203, and my user name is mjones. Our company is based in Tampa, Florida." + + # Create an agent using LLMfx class + agent = LLMfx() + + # Load the work + agent.load_work(customer_transcript) + + # Load tools individually + agent.load_tool("sentiment") + agent.load_tool("ner") + + # Load multiple tools + agent.load_tool_list(["emotions", "topics", "intent", "tags", "ratings", "answer"]) + + # Start deploying tools and running various analytics + # First, conduct three 'soft skills' initial assessment using 3 different models + agent.sentiment() + agent.emotions() + agent.intent() + + # Alternative way to execute a tool, passing the tool name as a string + agent.exec_function_call("ratings") + + # Call multiple tools concurrently + agent.exec_multitool_function_call(["ner", "topics", "tags"]) + + # The 'answer' tool is a quantized question-answering model - ask an 'inline' question + # The optional 'key' assigns the output to a dictionary key for easy consolidation + agent.answer("What is a short summary?", key="summary") + + # Prompting tool to ask a quick question as part of the analytics + response = agent.answer("What is the customer's account number and user name?", key="customer_info") + + # You can 'unload_tool' to release it from memory + agent.unload_tool("ner") + agent.unload_tool("topics") + + # At the end of processing, show the report that was automatically aggregated by key + report = agent.show_report() + + # Display a summary of the activity in the process + activity_summary = agent.activity_summary() + + # List of the responses gathered + for i, entries in enumerate(agent.response_list): + print(f"Update: response analysis {i}: {entries}") + assert entries is not None + + assert activity_summary is not None + assert agent.journal is not None + assert report is not None + + output = { + "report": report, + "activity_summary": activity_summary, + "journal": agent.journal + } + + return output diff --git a/tests/models/test_cloud_model_providers.py b/tests/models/test_cloud_model_providers.py new file mode 100644 index 0000000..189e2e5 --- /dev/null +++ b/tests/models/test_cloud_model_providers.py @@ -0,0 +1,42 @@ +""" Basic connectivity tests to cloud API providers. """ + +import os +from llmware.prompts import Prompt + +openai_api_key = os.environ.get("OPENAI_API_KEY","") +anthropic_api_key = os.environ.get("ANTHROPIC_API_KEY","") +ai21_api_key = os.environ.get("AI21_API_KEY","") +google_api_key = os.environ.get("GOOGLE_API_KEY","") +cohere_api_key = os.environ.get("COHERE_API_KEY", "") + + +# Simple test to make sure we are reaching OpenAI +def test_openai(): + prompter = Prompt(llm_name="gpt-4", llm_api_key=openai_api_key) + response = prompter.completion("what is artificial intelligence?") + llm_response = response["llm_response"] + assert 'artificial' in llm_response.lower() + + +# Simple test to make sure we are reaching Google +def test_google(): + prompter = Prompt(llm_name="text-bison@001", llm_api_key=google_api_key) + response = prompter.completion("what is artificial intelligence?") + llm_response = response["llm_response"] + assert 'artificial' in llm_response.lower() + + +# Simple test to make sure we are reaching Anthropic +def test_anthropic(): + prompter = Prompt(llm_name="claude-instant-v1", llm_api_key=anthropic_api_key) + response = prompter.completion("what is artificial intelligence?") + llm_response = response["llm_response"] + assert 'artificial' in llm_response.lower() + + +# Simple test to make sure we are reaching AI21 +def test_ai21(): + prompter = Prompt(llm_name="j2-grande-instruct", llm_api_key=ai21_api_key) + response = prompter.completion("what is artificial intelligence?") + llm_response = response["llm_response"] + assert 'artificial' in llm_response.lower() diff --git a/tests/models/test_cohere_command_r_model.py b/tests/models/test_cohere_command_r_model.py new file mode 100644 index 0000000..749dbdb --- /dev/null +++ b/tests/models/test_cohere_command_r_model.py @@ -0,0 +1,29 @@ +""" Test for Cohere Command R model integration. """ + +from llmware.models import ModelCatalog + +def test_cohere_command_r(): + """ Test loading and basic inference for Cohere Command R model. """ + + model_name = "command-r" + + # Load the model using ModelCatalog + model = ModelCatalog().load_model(model_name) + + # Define a simple prompt for testing + prompt = "What is the capital of France?" + + # Perform inference + response = model.inference(prompt) + + # Print the response for debugging + print("LLM Response:", response["llm_response"]) + print("Usage:", response["usage"]) + + # Assert that the response is not None + assert response is not None + + return 0 + +if __name__ == "__main__": + test_cohere_command_r() diff --git a/tests/models/test_gguf_model_load.py b/tests/models/test_gguf_model_load.py new file mode 100644 index 0000000..c0ce3b0 --- /dev/null +++ b/tests/models/test_gguf_model_load.py @@ -0,0 +1,49 @@ + +""" Test that GGUF models are loading correctly in local environment. By default, will run through a series of + different GGUF models in the ModelCatalog to spot-check that the model is correctly loading and + successfully completing an inference: + + # tests several different underlying models: + + # bling-answer-tool -> tiny-llama (1b) + # bling-phi-3-gguf -> phi-3 (3.8b) + # dragon-yi-answer-tool -> yi (6b) + # dragon-llama-answer-tool -> llama-2 (7b) + # llama-2-7b-chat-gguf -> llama-2-chat (7b) + # dragon-mistral-answer-tool -> mistral-1 (7b) + + """ + + +from llmware.models import ModelCatalog + + +def test_gguf_model_load(): + + # feel free to adapt this model list + + model_list = ["bling-answer-tool", + "bling-phi-3-gguf", + "dragon-yi-answer-tool", + "dragon-llama-answer-tool", + "llama-2-7b-chat-gguf", + "dragon-mistral-answer-tool"] + + # please note that the unusually short and simple prompt at times actually yields more variability in the model + # response - we are only testing for successful loading and inference + + sample_prompt = ("The company stock declined by $12 after poor earnings results." + "\nHow much did the stock price decline?") + + for model_name in model_list: + + print("\nmodel name: ", model_name) + + model = ModelCatalog().load_model(model_name, temperature=0.0, sample=False) + + response = model.inference(sample_prompt) + + print(f"{model_name} - response: ", response) + + assert response is not None + diff --git a/tests/models/test_hf_model_load_prompt.py b/tests/models/test_hf_model_load_prompt.py new file mode 100644 index 0000000..afd5773 --- /dev/null +++ b/tests/models/test_hf_model_load_prompt.py @@ -0,0 +1,32 @@ + + +from llmware.models import ModelCatalog +from llmware.prompts import Prompt + + +def test_load_hf_model_in_prompt(): + + model_list = ["llmware/bling-1b-0.1", + "llmware/bling-1.4b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/bling-falcon-1b-0.1"] + + test_prompt = ("The best time to visit New York City is in the Fall when the weather is nicest." + "\nWhen is the best time to visit New York?") + + for model_name in model_list: + + print("model_name: ", model_name) + + prompter = Prompt().load_model(model_name, temperature=0.0, sample=False) + assert prompter is not None + + response = prompter.prompt_main(test_prompt) + + print(f"{model_name} response - ", response) + + assert response is not None + + + +test_load_hf_model_in_prompt() diff --git a/tests/models/test_prompt_benchmark_test.py b/tests/models/test_prompt_benchmark_test.py new file mode 100644 index 0000000..ab9b07f --- /dev/null +++ b/tests/models/test_prompt_benchmark_test.py @@ -0,0 +1,128 @@ +"""This runs a benchmark test dataset against a series of prompts. It can be used to test any model type for +longer running series of prompts, as well as the fact-checking capability. + +This test uses the RAG Benchmark test set, which can be pulled down from the LLMWare repository on +Huggingface at: www.huggingface.co/llmware/rag_instruct_benchmark_tester, or by using the +datasets library, which can be installed with: + + `pip3 install datasets` +""" + +import time +import random +import logging +import numpy as np +import matplotlib.pyplot as plt +from llmware.prompts import Prompt + +# The datasets package is not installed automatically by llmware +try: + from datasets import load_dataset +except ImportError: + raise ImportError("This test requires the 'datasets' Python package. " + "You can install it with 'pip3 install datasets'") + +# Set up logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +def load_rag_benchmark_tester_dataset(): + """Loads benchmark dataset used in the prompt test.""" + dataset_name = "llmware/rag_instruct_benchmark_tester" + logging.info(f"Loading RAG dataset '{dataset_name}'...") + dataset = load_dataset(dataset_name) + + test_set = [] + for i, samples in enumerate(dataset["train"]): + test_set.append(samples) + + return test_set + +def load_models(models): + """Load a list of models dynamically.""" + for model in models: + try: + logging.info(f"Loading model '{model}'") + yield Prompt().load_model(model) + except Exception as e: + logging.error(f"Failed to load model '{model}': {e}") + +def test_prompt_rag_benchmark(selected_test_models): + test_dataset = load_rag_benchmark_tester_dataset() + + # Randomly select one model from the list + r = random.randint(0, len(selected_test_models) - 1) + model_name = selected_test_models[r] + + logging.info(f"Selected model: {model_name}") + prompter = next(load_models([model_name])) + + logging.info(f"Running RAG Benchmark Test against '{model_name}' - 200 questions") + results = [] + for i, entry in enumerate(test_dataset): + try: + start_time = time.time() + + prompt = entry["query"] + context = entry["context"] + response = prompter.prompt_main(prompt, context=context, prompt_name="default_with_context", temperature=0.3) + + assert response is not None + + # Print results + time_taken = round(time.time() - start_time, 2) + logging.info(f"{i + 1}. llm_response - {response['llm_response']}") + logging.info(f"{i + 1}. gold_answer - {entry['answer']}") + logging.info(f"{i + 1}. time_taken - {time_taken}") + + # Fact checking + fc = prompter.evidence_check_numbers(response) + sc = prompter.evidence_comparison_stats(response) + sr = prompter.evidence_check_sources(response) + + for fc_entry in fc: + for f, facts in enumerate(fc_entry["fact_check"]): + logging.info(f"{i + 1}. fact_check - {f} {facts}") + + for sc_entry in sc: + logging.info(f"{i + 1}. comparison_stats - {sc_entry['comparison_stats']}") + + for sr_entry in sr: + for s, source in enumerate(sr_entry["source_review"]): + logging.info(f"{i + 1}. source - {s} {source}") + + results.append({ + "llm_response": response["llm_response"], + "gold_answer": entry["answer"], + "time_taken": time_taken, + "fact_check": fc, + "comparison_stats": sc, + "source_review": sr + }) + + except Exception as e: + logging.error(f"Error processing entry {i}: {e}") + + # Performance metrics + total_time = sum(result["time_taken"] for result in results) + average_time = total_time / len(results) if results else 0 + logging.info(f"Total time taken: {total_time} seconds") + logging.info(f"Average time per question: {average_time} seconds") + + # Visualization + time_taken_list = [result["time_taken"] for result in results] + plt.plot(range(1, len(time_taken_list) + 1), time_taken_list, marker='o') + plt.xlabel('Question Number') + plt.ylabel('Time Taken (seconds)') + plt.title('Time Taken per Question') + plt.show() + + return results + +# Example usage +if __name__ == "__main__": + selected_test_models = [ + "llmware/bling-1b-0.1", "llmware/bling-1.4b-0.1", "llmware/bling-falcon-1b-0.1", + "llmware/bling-tiny-llama-v0", "bling-phi-3-gguf", "bling-answer-tool", + "dragon-yi-answer-tool", "dragon-llama-answer-tool", "dragon-mistral-answer-tool" + ] + test_prompt_rag_benchmark(selected_test_models) diff --git a/tests/models/test_slim_fx_model_load.py b/tests/models/test_slim_fx_model_load.py new file mode 100644 index 0000000..6c17292 --- /dev/null +++ b/tests/models/test_slim_fx_model_load.py @@ -0,0 +1,35 @@ + +""" Test that 'SLIM' tool GGUF models are loading correctly in local environment. By default, will run + through a series of automated tests packaged with each model. Feel free to select among shorter list + of models - generally, if one model is working than likely the rest will work as well. """ + + +from llmware.models import ModelCatalog + + +def slim_tests(): + + """ Each of these one line commands will locally cache the model and then run a series of tests using + the model to demonstrate its use and confirm that installation locally was successfully. """ + + # running automated tests - see the tools in action + + tools= ["slim-extract-tool", "slim-xsum-tool", "slim-summary-tool", "slim-boolean-tool", + "slim-sentiment-tool" , "slim-topics-tool", "slim-ner-tool", "slim-ratings-tool", + "slim-emotions-tool", "slim-intent-tool", "slim-tags-tool", "slim-sql-tool", + "slim-category-tool", "slim-nli-tool", "slim-sa-ner-tool", "slim-tags-3b-tool"] + + small_tool_list = ["slim-extract-tool", "slim-topics-tool"] + + # run tests for one tool + ModelCatalog().tool_test_run("slim-sentiment-tool") + + # run tests for a bunch of tools - try the 'small_tool_list' as alternative + for tool in tools: + # excluding sentiment, since ran above as separate test + if tool != "slim-sentiment-tool": + ModelCatalog().tool_test_run(tool) + + return 0 + + diff --git a/tests/models/test_whisper_cpp_model_load.py b/tests/models/test_whisper_cpp_model_load.py new file mode 100644 index 0000000..99d68bf --- /dev/null +++ b/tests/models/test_whisper_cpp_model_load.py @@ -0,0 +1,58 @@ + +""" This tests WhisperCPP deployment and model loading. """ + + +import os +from llmware.models import ModelCatalog +from llmware.gguf_configs import GGUFConfigs +from llmware.setup import Setup + +# optional / to adjust various log/display parameters of the model +GGUFConfigs().set_config("whisper_cpp_verbose", "OFF") +GGUFConfigs().set_config("whisper_cpp_realtime_display", True) + +# note: english is default output - change to 'es' | 'fr' | 'de' | 'it' ... +GGUFConfigs().set_config("whisper_language", "en") + +# whether to add or remove segment markers in llm response output +GGUFConfigs().set_config("whisper_remove_segment_markers", True) + + +def test_whisper_cpp(): + + """ Execute a basic inference on Voice-to-Text model passing a file_path string """ + + voice_samples = Setup().load_voice_sample_files(small_only=True, over_write=True) + + example = "famous_quotes" + + fp = os.path.join(voice_samples,example) + + files = os.listdir(fp) + + # these are the two key lines + whisper_base_english = "whisper-cpp-base-english" + + model = ModelCatalog().load_model(whisper_base_english) + + for f in files: + + if f.endswith(".wav"): + + prompt = os.path.join(fp,f) + + print(f"\n\nPROCESSING: prompt = {prompt}") + + response = model.inference(prompt) + + print("\nllm response: ", response["llm_response"]) + print("usage: ", response["usage"]) + + assert response is not None + + return 0 + + + + + diff --git a/tests/retrieval/test_search_in_memory.py b/tests/retrieval/test_search_in_memory.py new file mode 100644 index 0000000..0eb5317 --- /dev/null +++ b/tests/retrieval/test_search_in_memory.py @@ -0,0 +1,33 @@ +import os +import sys +from llmware.parsers import Parser +from llmware.setup import Setup +from llmware.util import Utilities + +sys.path.append(os.path.join(os.path.dirname(__file__),"..")) +from utils import Logger + + +def test_parse_and_search_in_memory(): + + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path, "Agreements") + + # mix of queries - empty, single token, multi-token + # --if query is empty, then all results returned + # --case insensitive + query_list = ["base salary", "governing law", "effective date", "target annual bonus", "nyx pan", "pan",""] + + for i, contract in enumerate(os.listdir(contracts_path)): + Logger().log(f"\n > Analyzing {contract}...") + if contract != ".DS_Store": + output = Parser().parse_one(contracts_path, contract) + for query in query_list: + Logger().log(f"\nQuery: {contract} {query}") + results = Utilities().fast_search_dicts(query, output, remove_stop_words=True) + + Logger().log("Results:") + for j, entry in enumerate(results): + Logger().log(f"{j}. {len(entry['text'])} {entry}") + +test_parse_and_search_in_memory() diff --git a/tests/run-tests.py b/tests/run-tests.py new file mode 100755 index 0000000..263538c --- /dev/null +++ b/tests/run-tests.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python +import os +import pytest +import shutil +import subprocess +import sys + +from utils import Logger + + +class RunTests: + # Initialize and grab the folder paths for the llmware and llmware-packaging repos + def __init__(self): + tests_folder = os.path.dirname(os.path.realpath(__file__)) + self.repo_root = os.path.abspath(os.path.join(tests_folder, "..")) + self.logger = Logger() + + # Ensure the latest llmware module is installed locally + def update_llmware_install(self): + self.logger.log("Updating llmware installation...") + try: + self.run_command("pip uninstall llmware -y", self.repo_root) + self.run_command("pip install .", self.repo_root) + except Exception as e: + self.logger.log(f"Error updating llmware: {e}", level="error") + sys.exit(1) + + # Clean the environment to start from a fresh state + def clean_the_environment(self): + self.logger.log("Cleaning the environment...") + + # Remove the default llmware data folders + self.remove_folder(os.path.join(os.environ["HOME"], "llmware_data")) + self.remove_folder(os.path.join(os.environ["HOME"], "llmware_data_custom")) + + # Reset MongoDB + try: + self.logger.log("Resetting MongoDB (dropping the 'llmware' database)...") + from llmware.resources import MongoDBManager + MongoDBManager().client.drop_database("llmware") + except Exception as e: + self.logger.log(f"Error resetting MongoDB: {e}", level="error") + sys.exit(1) + + # Reset Milvus collections + try: + self.logger.log("Resetting Milvus (dropping all collections)...") + from llmware.configs import LLMWareConfig + from pymilvus import connections, utility + connections.connect("default", host=os.environ.get('MILVUS_HOST', 'localhost'), port=19530) + for collection in utility.list_collections(): + utility.drop_collection(collection) + self.logger.log("All Milvus collections dropped successfully.") + except Exception as e: + self.logger.log(f"Error resetting Milvus: {e}", level="error") + sys.exit(1) + + # Helper function to remove a folder if it exists + def remove_folder(self, folder_path): + if os.path.exists(folder_path): + try: + self.logger.log(f"Removing folder: {folder_path}") + shutil.rmtree(folder_path) + except Exception as e: + self.logger.log(f"Error removing folder {folder_path}: {e}", level="error") + else: + self.logger.log(f"Folder not found: {folder_path}") + + # Run a system command in the given directory, wait for it to complete, and log the output + def run_command(self, command, working_dir): + self.logger.log(f"Running command '{command}' in '{working_dir}'...") + command_array = command.split(" ") + try: + process = subprocess.Popen(command_array, cwd=working_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = process.communicate() + if stdout: + self.logger.log(stdout.decode('utf-8')) + if stderr: + self.logger.log(stderr.decode('utf-8'), level="error") + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, command) + except subprocess.CalledProcessError as e: + self.logger.log(f"Command '{command}' failed with error: {e}", level="error") + sys.exit(1) + except Exception as e: + self.logger.log(f"An unexpected error occurred while running '{command}': {e}", level="error") + sys.exit(1) + + +if __name__ == "__main__": + test_runner = RunTests() + + # Update and clean environment before running tests + test_runner.update_llmware_install() + test_runner.clean_the_environment() + + # Run the tests with pytest + try: + pytest.main(sys.argv[1:]) + except Exception as e: + test_runner.logger.log(f"Error running tests: {e}", level="error") + sys.exit(1) diff --git a/tests/set-env.sh b/tests/set-env.sh new file mode 100644 index 0000000..3f545e3 --- /dev/null +++ b/tests/set-env.sh @@ -0,0 +1,22 @@ +# LLM API KEYS +#export OPENAI_API_KEY= +#export ANTHROPIC_API_KEY= +#export AI21_API_KEY= +#export COHERE_API_KEY= +#export READ_GPT_API_KEY= +#export GOOGLE_API_KEY= + +# DB KEYS AND CONNCTION STRINGS +#export COLLECTION_DB_URI= +#export MONGO_ATLAS_CONNECTION_URI= + +#export MILVUS_HOST= +#export MILVUS_PORT= + +#export PINECONE_API_KEY= +#export PINECONE_ENVIRONMENT= + +#export NEO4J_URI= +#export NEO4J_USERNAME= +#export NEO4J_PASSWORD= +#export NEO4J_DATABASE= diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..6086a5d --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,14 @@ +from tabulate import tabulate + +class Logger(): + def __init__(self): + self.MAGENTA = '\033[95m' + self.END = '\033[0m' + self.BOLD = '\033[1m' + self.UNDERLINE = '\033[4m' + + def log(self, message): + print (self.MAGENTA + message + self.END) + + def log_table(self, headers, data): + self.log(tabulate(data, headers=headers, tablefmt="grid")) diff --git a/tutorials/getting_started/README.md b/tutorials/getting_started/README.md new file mode 100644 index 0000000..0661abc --- /dev/null +++ b/tutorials/getting_started/README.md @@ -0,0 +1,45 @@ + 🚀 Getting Set up - Key Items 🚀 +=============== + +**How to get started with LLMWare?** + +In this repository, we have small code snippets with a lot of annotation that describe key setup items to get started quickly and successfully with LLMWare: + +1. [**Configuring a DB**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/configure_db.py) - for a fast start, you don't have to do anything except set the .active_db parameter to "sqlite" and all libraries will be written on a local embedded sqlite instance with no installation required. + + + from llmware.configs import LLMWareConfig + LLMWareConfig().set_active_db("sqlite") + + + After writing content into your first library, you will find the sqlite db at the path defined here: + + from llmware.configs import SQLiteConfig + SQLiteConfig().get_config("sqlite_db_folder_path") + + +2. [**Loading Sample Files**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/loading_sample_files.py) - for an overview of the sample files available and how to pull down (integrated into many examples too.) + + + from llmware.setup import Setup + Setup().load_sample_files() + + +3. [**Using the Model Catalog**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/using_the_model_catalog.py) - this is the primary model discovery and loading class for LLMWare. + + + from llmware.models import ModelCatalog + all_models = ModelCatalog().list_all_models() + for model in all_models: print("model: ", model) + + +4. [**Working with Libraries**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/working_with_libraries.py) - Libraries are the main organizing construct for text collections in LLMWare. + + +As you are getting started, we would also recommend that you check out the [**Fast Start**](https://www.github.com/llmware-ai/llmware/tree/main/fast_start) examples and videos as well! + + + +### **Let's get started! 🚀** + + diff --git a/tutorials/getting_started/configure_db.py b/tutorials/getting_started/configure_db.py new file mode 100644 index 0000000..26c92ce --- /dev/null +++ b/tutorials/getting_started/configure_db.py @@ -0,0 +1,45 @@ + +""" This example shows how to select and set the collection database. The collection DB is used as the primary +source of storing text chunks from parsing runs, and organizing into Libraries. LLMWare supports three +collection databases: MongoDB, Postgres, and SQLite. + + For a fast no-install start, we would recommend SQLite. + + If you prefer SQL and massive scalability, we would recommend Postgres. + + If you prefer no-SQL and combo of scale and flexibility, we would recommend MongoDB. + + The LLMWare functions, methods and interfaces to each of the DBs are exactly the same with a + high-level abstraction interface provided within LLMWare CollectionRetrieval and CollectionWriter classes + that handle the implementation-specific details of writing and retrieving from each of these data sources. + + While semantic retrieval requires the use of a separate vector database and creating an embedding for the + content in the Library, once the library is created, text search retrieval is available automatically so text + Queries can be run immediately after Parsing. + +""" + +from llmware.configs import LLMWareConfig + + +def set_collection_database(): + + # check the current active db + active_db = LLMWareConfig().get_active_db() + + print("update: current 'active' collection database - ", active_db) + + # supported db list + supported_db = LLMWareConfig().get_supported_collection_db() + + print("update: supported collection databases - ", supported_db) + + # set the current active db -> once set, any calls to library / parsing will write to this db + LLMWareConfig.set_active_db("sqlite") + + # change anytime -> going forward, any new libraries will be written to the new db, but existing libraries + # remain on the previous db + LLMWareConfig.set_active_db("postgres") + + return 0 + diff --git a/tutorials/getting_started/loading_sample_files.py b/tutorials/getting_started/loading_sample_files.py new file mode 100644 index 0000000..7686307 --- /dev/null +++ b/tutorials/getting_started/loading_sample_files.py @@ -0,0 +1,94 @@ + +""" To enable rapid testing and prototyping, LLMWare provides a range of sample files that can be accessed +through the Setup class, and specific methods that will pull the files from a non-restricted llmware AWS S3 bucket, +and download the files locally in the /llmware_data/sample_files path. + + Sample files are created from public domain sources, and often developed originally by LLMWare. They are +provided for convenience in testing. As we find or develop a good testing set, we will generally try to make it +available so everyone can use - as a result, the sample files collection is evolving and growing all the time! + + If you have an older version, and would like to get the latest, then you can set the over_write=True option. + +""" + +import os +from llmware.setup import Setup + + +def get_llmware_sample_files(): + + print (f"\n > Loading the llmware Sample Files...") + + # this call to Setup() will pull the sample files from a public S3 repo + # the files will be placed in folders locally at the sample_files_path + + sample_files_path = Setup().load_sample_files(over_write=False) + + print(f"> Sample Files Path - {sample_files_path}") + + print(f"> Sample Folders - {os.listdir(sample_files_path)}") + + # Current Sample Files: + # + # AgreementsLarge = ~80 sample contracts + # Agreements = ~15 sample employment agreements + # UN-Resolutions-500 = 500 United Nations Resolutions (~2 years) - public repo - PDF files + # Invoices = ~40 invoice sample documents + # FinDocs = ~15 financial annual reports, earnings and 10Ks + # AWS-Transcribe = ~5 AWS-transcribe JSON files + # SmallLibrary = ~10 mixed document types for quick testing + # Images = ~3 images for OCR processing + + # Note: these files will be updated from time-to-time - if you want to pull fresh new files + # sample_files_path = Setup().load_sample_files(over_write=True) + + # These files were prepared by LLMWare from public domain materials or invented bespoke as examples + # If you have any concerns about PII or the suitability or any material, please let us know + # We reserve the right to withdraw documents at any time + + return 0 + + +def get_llmware_voice_sample_files(): + + print (f"\n > Loading the llmware Sample Voice Files...") + + # check out the examples: Models/using-whisper-cpp-sample-files.py and + # Use_Cases/parsing_great_speeches.py + + # this call to Setup() will pull the sample files from a public S3 repo + # the files will be placed in folders locally at the sample voice files path + + sample_files_path = Setup().load_voice_sample_files(over_write=False,small_only=False) + + print(f"> Sample Voice Files Path - {sample_files_path}") + + print(f"> Sample Voice Folders - {os.listdir(sample_files_path)}") + + # examples - "famous_quotes" | "greatest_speeches" | "youtube_demos" | "earnings_calls" + + # -- famous_quotes - approximately 20 small .wav files with clips from old movies and speeches + # -- greatest_speeches - approximately 60 famous historical speeches in english + # -- youtube_videos - wav files of ~3 llmware youtube videos + # -- earnings_calls - wav files of ~4 public company earnings calls (gathered from public investor relations) + + # These sample files are hosted in a non-restricted AWS S3 bucket, and downloaded via the Setup method + # `load_sample_voice_files`. There are two options: + + # -- small_only = True: only pulls the 'famous_quotes' samples + # -- small_only = False: pulls all of the samples (requires ~1.9 GB in total) + + # Please note that all of these samples have been pulled from open public domain sources, including the + # Internet Archives, e.g., https://archive.org. These sample files are being provided solely for the purpose of + # testing the code scripts below. Please do not use them for any other purpose. + + return 0 + + +if __name__ == "__main__": + + get_llmware_sample_files() + + get_llmware_voice_sample_files() + + diff --git a/tutorials/getting_started/using_the_model_catalog.py b/tutorials/getting_started/using_the_model_catalog.py new file mode 100644 index 0000000..c1f7b7d --- /dev/null +++ b/tutorials/getting_started/using_the_model_catalog.py @@ -0,0 +1,78 @@ + +""" This example shows how to get started using the ModelCatalog to instantiate models and start running inferences. + + A core design principle of LLMWare is that all models should be accessible in the same way, with the + underlying configuration details handled at lower levels of the implementation. Our aspiration is that you + should be able to substitute models in any pipeline/workflow at any point with minimal, if any, code change - + and to be able to deploy heterogeneous combinations of local, private network and API-based models, + with mix-qnd-match and full 'swqp-ability' at any time. + + While LLMWare supports OpenAI and other API-based models, we are "open-source-first" in our view and + support of models, and all of our examples and classes/methods are optimized first to work with small, specialized + open source models. Our view is that essentially anything that can be done with large API models can be + replicated with smaller fine-tuned models and well-designed data pipelines and workflows. + + We provide over 50+ LLMWare models that are all tested and designed for easy integration + into LLMWare pipeline and workflows, but we are 100% committed to providing an open ecosystem and supporting + the best in open source, including leading embedding models from Jina and Nomic, Llama-2, Llama-3, Mistral, + Phi-3, Yi, DeciLM, StableLM, OpenHermes, Tiny Llama, RedPajama, Pythia, Sheared LLama, Huggingface Zephyr, + SentenceTransformers and quantizations from The Bloke and Bartowski. We also provide a full integration into GGUF, + LLama.CPP and Whisper.CPP. + + It is easy to extend the ModelCatalog to include any of your favorite models from + HuggingFace, Ollama, LMStudio, LLama.CPP, or llama-cpp-python. + + For open source models, we generally support both PyTorch (Huggingface) based models, as well as GGUF (LLama.CPP) + quantized models. For most use cases, we find that GGUF quantized versions are faster, take less memory + and are generally as accurate, so we use GGUF-based models, aka "tools", in many of our examples. + + """ + +from llmware.models import ModelCatalog + +all_models = ModelCatalog().list_all_models() + +# ~133 models in the catalog as of May 4, 2024 - and growing all the time! +print("\nAll Models") + +for i, model in enumerate(all_models): + print("models: ", i, model) + +# ~30 embedding models - including Nomic, Jina, leading sentence transformers, llmware industry-bert models +embedding_models = ModelCatalog().list_embedding_models() + +""" +print("\nEmbedding Models") +for i, model in enumerate(embedding_models): + print("embedding models: ", i, model) +""" + +# ~92 open source models +open_source_models = ModelCatalog().list_open_source_models() + +""" +print("\nOpen Source Models") +for i, model in enumerate(open_source_models): + print("open source models: ", i, model) +""" + +# Inference is Easy - same two lines every time +# 1. load_model - use the model_name or display_name, and it will be looked up and instantiated +# 2. inference - all models support an inference method - call it to run a basic inference + +model_name = "phi-3-gguf" +model = ModelCatalog().load_model(model_name) +response = model.inference("I am going to Paris. What should I see?") + +print(f"\ntest inference - {model_name} - response: ", response) + +# Check out other examples in Models, Prompts, SLIM-Agents Embeddings, and Use_Cases +# -- configuration in loading model - temperature, sample, max_output +# -- add models to the Catalog for easy invocation +# -- integrating sources and building RAG workflows in Prompts +# -- fact-checking and post-processing +# -- using function-calls on small specialized models +# -- agent based workflows +# -- installing embeddings on a library collection + + diff --git a/tutorials/getting_started/welcome_example.py b/tutorials/getting_started/welcome_example.py new file mode 100644 index 0000000..3d7afe1 --- /dev/null +++ b/tutorials/getting_started/welcome_example.py @@ -0,0 +1,72 @@ + +""" This 'welcome_example' can serve as a quick 'hello world' test to verify that LLMWare has been installed +and that local model inference serving over GGUF is available and working as expected. This example will +pull down two models and run a quick 'Welcome' for a new user. """ + + +from llmware.models import ModelCatalog +from llmware.configs import LLMWareConfig + +# optional - adds a dash of color to the console output +try: + from colorama import Fore + BLUE = Fore.BLUE + RESET = Fore.RESET +except: + BLUE = "" + RESET = "" + +print(f"\n{BLUE}Welcome to LLMWare - Test Script{RESET}") + +# run first inference +print(f"\nLoading {BLUE}bling-phi-3-gguf model{RESET} for First Inference: may take a minute to download the first time") +print(f"Model will be cached at the local path: {BLUE}{LLMWareConfig().get_model_repo_path()}{RESET}") + +try: + + # loads the model from the model catalog + model = ModelCatalog().load_model("bling-phi-3-gguf", temperature=0.0, sample=False) + + prompt = ("When this script is loaded, we should greet the person by saying, 'Welcome to LLMWare.'" + "\nThis script has been loaded - what should we say?") + + print(f"\nprompt: {prompt}") + + # executes inference + response = model.inference(prompt) + + print("\nmodel response: ", response) + +except: + print("\nWe are sorry but something has gone wrong with the installation and/or download of the file, and it " + "did not start correctly. Please check the documentation.\nMost common sources of this error: " + "\n1. Supported platforms - Mac M1/M2/M3, Linux x86, Windows" + "\n2. Model did not download correctly. Try again and confirm that model instantiated in local folder path." + "\n3. Update/pull from the latest repo and/or the latest pip install version.") + + +# run second inference +print(f"\nLoading {BLUE}slim-sentiment-tool{RESET}") + +try: + + # load the model from the model catalog + model = ModelCatalog().load_model("slim-sentiment-tool", temperature=0.0, sample=False) + + user = "We are very happy and excited to get started with LLMWare." + print(f"\nuser: {user}") + + # execute a 'sentiment' function call on the model + response = model.function_call(user, function="classify", params=["sentiment"],get_logits=False) + + print("\nclassifying the sentiment: ", response) + +except: + print("We are sorry but something has gone wrong with the installation and/or download of the file, and it " + "did not start correctly. Please check the documentation.\nMost common sources of this error: " + "\n1. Supported platforms - Mac M1/M2/M3, Linux x86, Windows" + "\n2. Model did not download correctly. Try again and confirm that model instantiated in local folder path." + "\n3. Update/pull from the latest repo and/or the latest pip install version.") + + +print(f"\n\nPlease review /examples for other examples.\n\n{BLUE}Welcome to the LLMWare community.{RESET}") diff --git a/tutorials/getting_started/working_with_libraries.py b/tutorials/getting_started/working_with_libraries.py new file mode 100644 index 0000000..21386d8 --- /dev/null +++ b/tutorials/getting_started/working_with_libraries.py @@ -0,0 +1,102 @@ + +""" This example shows the basics of working with Libraries. Library is the main organizing construct for +unstructured information in LLMWare. Users can create one large library with all types of different content, or +can create multiple libraries with each library comprising a specific logical collection of information on a +particular subject matter, project/case/deal, or even different accounts/users/departments. + + Each Library consists of the following components: + + 1. Collection on a Database - this is the core of the Library, and is created through parsing of documents, which + are then automatically chunked and indexed in a text collection database. This is the basis for retrieval, + and the collection that will be used as the basis for tracking any number of vector embeddings that can be + attached to a library collection. + + 2. File archives - found in the llmware_data path, within Accounts, there is a folder structure for each Library. + All file-based artifacts for the Library are organized in these folders, including copies of all files added in + the library (very useful for retrieval-based applications), images extracted and indexed from the source + documents, as well as derived artifacts such as nlp and knowledge graph and datasets. + + 3. Library Catalog - each Library is registered in the LibraryCatalog table, with a unique library_card that has + the key attributes and statistics of the Library. + + When a Library object is passed to the Parser, the parser will automatically route all information into the + Library structure. + + The Library also exposes convenience methods to easily install embeddings on a library, including tracking of + incremental progress. + + To parse into a Library, there is the very useful convenience methods, "add_files" which will invoke the Parser, + collate and route the files within a selected folder path, check for duplicate files, execute the parsing, + text chunking and insertion into the database, and update all of the Library state automatically. + + Libraries are the main index constructs that are used in executing a Query. Pass the library object when + constructing the Query object, and then all retrievals (text, semantic and hybrid) will be executed against + the content in that Library only. + +""" + +import json +import os +from llmware.configs import LLMWareConfig +from llmware.library import Library, LibraryCatalog +from llmware.setup import Setup + + +def core_library_functions(library_name): + + # Create a library + print (f"\n > Creating library {library_name}...") + + library = Library().create_new_library(library_name) + + # Load an existing library. This not required after library creation and is only shown here for reference + library = Library().load_library(library_name) + + # The LibraryCatalog is used to query all libraries + print (f"\n > All libraries") + for library_card in LibraryCatalog().all_library_cards(): + lib_name = library_card["library_name"] + docs = library_card["documents"] + print (f" - {lib_name} ({docs} documents)") + + # Add a few files to our library + print (f"\n > Adding some files to {library_name}") + sample_files_path = Setup().load_sample_files() + library.add_files(os.path.join(sample_files_path,"SmallLibrary")) + + # View the library card to confirm document, block and other counts + print (f"\n > Library Card") + library_card = library.get_library_card() + library_card["_id"] = str(library_card["_id"]) # The _id needs to be converted to a str before printing + print (json.dumps(library_card, indent=2)) + + # Library Export to JSON + print (f"\n > Exporting library to jsonl file...") + temp_export_dir = LLMWareConfig().get_tmp_path() + json_export_path = library.export_library_to_jsonl_file(temp_export_dir, "lib_export") + print (f" - library exported to {json_export_path}") + + # Library export to txt file + print (f"\n > Exporting library to text file...") + text_export_path = library.export_library_to_txt_file(temp_export_dir, "lib_export") + print (f" - library exported to {text_export_path}") + + # check out the images extracted + print (f"\n images extracted and saved at local path - {library.image_path}") + + # Delete the library + # print (f"\n > Deleting the library...") + # note: for safety: set the confirm_delete=True when/if you want to delete + library.delete_library(confirm_delete=False) + + +if __name__ == "__main__": + + # set the database to be used to store Library text collection + # 3 supported options: MongoDB, Postgres, SQLite + # MongoDB and Postgres require separate install, while SQLite is no-install/setup + + LLMWareConfig().set_active_db("sqlite") + + core_library_functions("library_tests2") + diff --git a/tutorials/notebooks/Agents_01.ipynb b/tutorials/notebooks/Agents_01.ipynb new file mode 100644 index 0000000..f27d5c9 --- /dev/null +++ b/tutorials/notebooks/Agents_01.ipynb @@ -0,0 +1,272 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "StkY5oHGU-iN" + }, + "source": [ + "# LLMWare Model Exploration\n", + "\n", + "## This is the 'entrypoint' example that provides a general introduction of llmware models.\n", + "\n", + "This notebook provides an introduction to LLMWare Agentic AI models and demonstrates their usage." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "KyaEnPzOVTJe" + }, + "outputs": [], + "source": [ + "# install dependencies\n", + "!pip3 install llmware" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mcOxXgs1XTjD" + }, + "source": [ + "If you have any dependency install issues, please review the README, docs link, or raise an Issue.\n", + "\n", + "Usually, if there is a missing dependency, the code will give the warning - and a clear direction like `pip install transformers'` required for this example, etc.\n", + "\n", + "As an alternative to pip install ... if you prefer, you can also clone the repo from github which provides a benefit of having access to 100+ examples.\n", + "\n", + "To clone the repo:\n", + "```\n", + "git clone \"https://www.github.com/llmware-ai/llmware.git\"\n", + "sh \"welcome_to_llmware.sh\"\n", + "```\n", + "\n", + "The second script `\"welcome_to_llmware.sh\"` will install all of the dependencies.\n", + "\n", + "If using Windows, then use the `\"welcome_to_llmware_windows.sh\"` script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "n4aKjcEiVjYE" + }, + "outputs": [], + "source": [ + "# Import Library\n", + "from llmware.models import ModelCatalog" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ePtRGBIlZEkP" + }, + "source": [ + "## GETTING STARTED WITH AGENTIC AI\n", + "All LLMWare models are accessible through the ModelCatalog generally consisting of two steps to access any model\n", + "\n", + "- Step 1 - load the model - pulls from global repo the first time, and then automatically caches locally\n", + "- Step 2 - use the model with inference or function call" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "D0xL5WOgVlGX" + }, + "outputs": [], + "source": [ + "# 'Standard' Models use 'inference' and take a general text input and provide a general text output\n", + "\n", + "model = ModelCatalog().load_model(\"bling-answer-tool\")\n", + "response = model.inference(\"My son is 21 years old.\\nHow old is my son?\")\n", + "\n", + "print(\"\\nresponse: \", response)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "1AkSZ3Z_VqWt" + }, + "outputs": [], + "source": [ + "# Optional parameters can improve results\n", + "model = ModelCatalog().load_model(\"bling-phi-3-gguf\", temperature=0.0,sample=False, max_output=200)\n", + "\n", + "# all LLMWare models have been fine-tuned to assume that the input will include a text passage, and that the\n", + "# model's main job is to 'read' the passage, and then 'answer' a question based on that information\n", + "\n", + "text_passage = \"The company's stock price increased by $3 after reporting positive earnings.\"\n", + "prompt = \"What was the increase in the stock price?\"\n", + "\n", + "response = model.inference(prompt,add_context=text_passage)\n", + "\n", + "print(\"\\nresponse: \", response)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OuNEktB-aPVw" + }, + "source": [ + "## Models we have and support\n", + "Inference models can also be integrated into Prompts - which provide advanced handling for integrating with knowledge retrieval, managing source information, and providing fact-checking\n", + "\n", + "Discovering other models is easy -> to invoke a model, simply use the `'model_name'` and pass in `.load_model()`.\n", + "\n", + "***note***: *model_names starting with `'bling'`, `'dragon'`, and `'slim'` are llmware models.*\n", + "- we do **include other popular models** such as `phi-3`, `qwen-2`, `yi`, `llama-3`, `mistral`\n", + "- it is easy to extend the model catalog to **include other 3rd party models**, including `ollama` and `lm studio`.\n", + "- we do **support** `open ai`, `anthropic`, `cohere` and `google api` models as well." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "erEHenbjaYqi" + }, + "outputs": [], + "source": [ + "all_generative_models = ModelCatalog().list_generative_local_models()\n", + "print(\"\\n\\nModel Catalog - load model with ModelCatalog().load_model(model_name)\")\n", + "for i, model in enumerate(all_generative_models):\n", + "\n", + " model_name = model[\"model_name\"]\n", + " model_family = model[\"model_family\"]\n", + "\n", + " print(\"model: \", i, model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tLCuxZcYdTHn" + }, + "source": [ + "## Slim Models\n", + "Slim models are 'Function Calling' Models that perform a specialized task and output python dictionaries\n", + "- by design, slim models are specialists that **perform single function**.\n", + "- by design, slim models generally **do not require any specific** `'prompt instructions'`, but will often accept a `\"parameter\"` which is passed to the function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true, + "id": "1ZS2wo8zdDOd" + }, + "outputs": [], + "source": [ + "model = ModelCatalog().load_model(\"slim-sentiment-tool\")\n", + "response = model.function_call(\"That was the worst earnings call ever - what a disaster.\")\n", + "\n", + "# the 'overall' model response is just a python dictionary\n", + "print(\"\\nresponse: \", response)\n", + "print(\"llm_response: \", response['llm_response'])\n", + "print(\"sentiment: \", response['llm_response']['sentiment'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ohp-shGjkDkz" + }, + "outputs": [], + "source": [ + "# here is one of the slim model applied against a common earnings extract\n", + "\n", + "text_passage = (\"Here’s what Costco reported for its fiscal second quarter of 2024 compared with what Wall Street \"\n", + " \"was expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: Earnings \"\n", + " \"per share: $3.92 vs. $3.62 expected. Revenue: $58.44 billion vs. $59.16 billion expected \"\n", + " \"In the three-month period that ended Feb. 18, Costco’s net income rose to $1.74 billion, or \"\n", + " \"$3.92 per share, compared with $1.47 billion, or $3.30 per share, a year earlier. \")\n", + "\n", + "# extract model takes a 'key' for a parameter, and looks for the 'value' in the text\n", + "model = ModelCatalog().load_model(\"slim-extract-tool\")\n", + "\n", + "# the general structure of a function call includes a text passage input, a function and parameters\n", + "response = model.function_call(text_passage,function=\"extract\",params=[\"revenue\"])\n", + "\n", + "print(\"\\nextract response: \", response)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "w13LLW_OdxCm" + }, + "outputs": [], + "source": [ + "# Function calling models generally come with a test set that is a great way to learn how they work\n", + "# please note that each test can take a few minutes with 20-40 test questions\n", + "\n", + "# You can try rest of them yourself by removing comment(s) of the below catalog.\n", + "ModelCatalog().tool_test_run(\"slim-topics-tool\")\n", + "ModelCatalog().tool_test_run(\"slim-tags-tool\")\n", + "ModelCatalog().tool_test_run(\"slim-emotions-tool\")\n", + "ModelCatalog().tool_test_run(\"slim-summary-tool\")\n", + "ModelCatalog().tool_test_run(\"slim-xsum-tool\")\n", + "ModelCatalog().tool_test_run(\"slim-boolean-tool\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kOPly8bfdnan" + }, + "source": [ + "## Agentic AI\n", + "Function calling models can be integrated into Agent processes which can orchestrate processes comprising multiple models and steps - most of our use cases will use the function calling models in that context\n", + "\n", + "## Last note:\n", + "Most of the models are packaged as `\"gguf\"` usually identified as GGUFGenerativeModel, or with `'-gguf'` or `'-tool` at the end of their name. These models are optimized to run most efficiently on a CPU-based laptop (especially Mac OS). You can also try the standard Pytorch versions of these models, which should yield virtually identical results, but will be slower." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rvLVgWYMe6RO" + }, + "source": [ + "## Journey is yet to start!\n", + "Loved it?? This is just an example of our models. Please check out our other Agentic AI examples with every model in detail here: https://github.com/llmware-ai/llmware/tree/main/fast_start/agents\n", + "\n", + "Also, if you have more interest in RAG, then please go with our RAG examples, which you can find here: https://github.com/llmware-ai/llmware/tree/main/fast_start/rag\n", + "\n", + "If you liked it, then please **star our repo https://github.com/llmware-ai/llmware** ⭐\n", + "\n", + "Any doubts?? Join our **discord server: https://discord.gg/GN49aWx2H3** 🫂" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/Agents_02.ipynb b/tutorials/notebooks/Agents_02.ipynb new file mode 100644 index 0000000..af663fa --- /dev/null +++ b/tutorials/notebooks/Agents_02.ipynb @@ -0,0 +1,375 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "XpQMDcKy-fa0" + }, + "source": [ + "# Welcome to BLING & DRAGON Model Zoo Sampler by LLMWare.ai 🦾\n", + "This is the 2nd tutorial of our Agents Fast Start Series. So please fasten your seat belts and get ready to dive in Bling and Dragon models.\n", + "\n", + "\n", + "## Get started with local inferences in minutes...\n", + "\n", + "This example is a \"hello world\" model zoo sampler for LLMWare BLING and DRAGON models, and shows a simple epeatable recipe for prompting models locally using a provided set of sample context passages and questions.\n", + "\n", + "It is designed to be easy to select among different LLMWARE models in the BLING (Best Little Instruct No-GPU) and DRAGON (Delivering RAG On ...) model families." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kWJY0nLC-fzM" + }, + "source": [ + "## But what are Bling and Dragon models??\n", + "Both Bling and Dragon models have been fine-tuned on complex business, financial and legal materials, with specific focus on accurate fact-based question-answering for a context passage.\n", + "\n", + "**BLING models** vary between 0.5B - 3.8B parameters and have been specifically designed to run on a CPU, including on local laptops.\n", + "\n", + "**DRAGON models** vary between 6-9B parameters, and when quantized, can generally run OK on most CPU-based laptops, but are designed for optimal use on a GPU or inference server. These models operate on the same principles as the BLING models, making it easy to 'test' with BLING, and then 'upgrade' to DRAGON in shifting into production environment for greater accuracy.\n", + "\n", + "### Key training objectives:\n", + "- **Fact Reliance**: Answers are grounded in the provided context; without it, the model often returns \"Not Found,\" reducing hallucinations and making it ideal for RAG and workflow tasks.\n", + "\n", + "- **Concise Answers**: Responses are short and factual—optimized for speed, integration, and automation rather than chat-like dialogue.\n", + "\n", + "- **Negative Sampling**: Trained to respond with \"Not Found\" when the context doesn’t contain the answer, aiding in multi-passage retrieval and filtering.\n", + "\n", + "- **Instruction-Free**: No system prompts or role instructions needed—just context and a question." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yU-iCADREXj4" + }, + "outputs": [], + "source": [ + "# install dependencies\n", + "!pip3 install llmware # if you're getting error, then try upgrading the pip by runnnig this command : python -m pip install --upgrade pip" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "MbFpA7pWEZok" + }, + "source": [ + "If you have any dependency install issues, please review the README, docs link, or raise an Issue.\n", + "\n", + "Usually, if there is a missing dependency, the code will give the warning - and a clear direction like `pip install transformers'` required for this example, etc.\n", + "\n", + "As an alternative to pip install ... if you prefer, you can also clone the repo from github which provides a benefit of having access to 100+ examples.\n", + "\n", + "To clone the repo:\n", + "```\n", + "git clone \"https://www.github.com/llmware-ai/llmware.git\"\n", + "sh \"welcome_to_llmware.sh\"\n", + "```\n", + "\n", + "The second script `\"welcome_to_llmware.sh\"` will install all of the dependencies.\n", + "\n", + "If using Windows, then use the `\"welcome_to_llmware_windows.sh\"` script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "otqo-DNMItUc" + }, + "outputs": [], + "source": [ + "# Now's let jump into the code part!\n", + "import time # To see how much time a model takes to load\n", + "from llmware.prompts import Prompt # A class (or function) inside the prompts module that helps you build or manage prompts for LLMs." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DpwHOs_ILM5d" + }, + "source": [ + "Now below is a function that contains a few test cases. You can add your own test cases as well.\n", + "\n", + "To adapt this test set, just create your own list with dictionary entries and keys 'query', 'answer' and 'context'." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "I8-bR172JT1K" + }, + "outputs": [], + "source": [ + "def hello_world_questions():\n", + "\n", + " \"\"\"\n", + " Representative test set - we would recommend running this script as a 'hello world' test on the first\n", + " try that you use a model, and then adapt the content to your own set of context and questions.\n", + "\n", + " There is nothing special about these questions, and in fact, you will note that many of the models will get\n", + " a couple of answers wrong (especially the ~1B parameter models). The errors are important insights\n", + " as you evaluate which models to consider for your use case.\n", + " \"\"\"\n", + "\n", + " test_list = [\n", + "\n", + " {\"query\": \"What is the total amount of the invoice?\",\n", + " \"answer\": \"$22,500.00\",\n", + " \"context\": \"Services Vendor Inc. \\n100 Elm Street Pleasantville, NY \\nTO Alpha Inc. 5900 1st Street \"\n", + " \"Los Angeles, CA \\nDescription Front End Engineering Service $5000.00 \\n Back End Engineering\"\n", + " \" Service $7500.00 \\n Quality Assurance Manager $10,000.00 \\n Total Amount $22,500.00 \\n\"\n", + " \"Make all checks payable to Services Vendor Inc. Payment is due within 30 days.\"\n", + " \"If you have any questions concerning this invoice, contact Bia Hermes. \"\n", + " \"THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000\"},\n", + "\n", + " {\"query\": \"What was the amount of the trade surplus?\",\n", + " \"answer\": \"62.4 billion yen ($416.6 million)\",\n", + " \"context\": \"Japan’s September trade balance swings into surplus, surprising expectations\"\n", + " \"Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, \"\n", + " \"beating expectations from economists polled by Reuters for a trade deficit of 42.5 \"\n", + " \"billion yen. Data from Japan’s customs agency revealed that exports in September \"\n", + " \"increased 4.3% year on year, while imports slid 16.3% compared to the same period \"\n", + " \"last year. According to FactSet, exports to Asia fell for the ninth straight month, \"\n", + " \"which reflected ongoing China weakness. Exports were supported by shipments to \"\n", + " \"Western markets, FactSet added. — Lim Hui Jie\"},\n", + "\n", + " {\"query\": \"What was Microsoft's revenue in the 3rd quarter?\",\n", + " \"answer\": \"$52.9 billion\",\n", + " \"context\": \"Microsoft Cloud Strength Drives Third Quarter Results \\nREDMOND, Wash. — April 25, 2023 — \"\n", + " \"Microsoft Corp. today announced the following results for the quarter ended March 31, 2023,\"\n", + " \" as compared to the corresponding period of last fiscal year:\\n· Revenue was $52.9 billion\"\n", + " \" and increased 7% (up 10% in constant currency)\\n· Operating income was $22.4 billion \"\n", + " \"and increased 10% (up 15% in constant currency)\\n· Net income was $18.3 billion and \"\n", + " \"increased 9% (up 14% in constant currency)\\n· Diluted earnings per share was $2.45 \"\n", + " \"and increased 10% (up 14% in constant currency).\\n\"},\n", + "\n", + " {\"query\": \"When did the LISP machine market collapse?\",\n", + " \"answer\": \"1987.\",\n", + " \"context\": \"The attendees became the leaders of AI research in the 1960s.\"\n", + " \" They and their students produced programs that the press described as 'astonishing': \"\n", + " \"computers were learning checkers strategies, solving word problems in algebra, \"\n", + " \"proving logical theorems and speaking English. By the middle of the 1960s, research in \"\n", + " \"the U.S. was heavily funded by the Department of Defense and laboratories had been \"\n", + " \"established around the world. Herbert Simon predicted, 'machines will be capable, \"\n", + " \"within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, \"\n", + " \"'within a generation ... the problem of creating 'artificial intelligence' will \"\n", + " \"substantially be solved'. They had, however, underestimated the difficulty of the problem. \"\n", + " \"Both the U.S. and British governments cut off exploratory research in response \"\n", + " \"to the criticism of Sir James Lighthill and ongoing pressure from the US Congress \"\n", + " \"to fund more productive projects. Minsky's and Papert's book Perceptrons was understood \"\n", + " \"as proving that artificial neural networks approach would never be useful for solving \"\n", + " \"real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period \"\n", + " \"when obtaining funding for AI projects was difficult, followed. In the early 1980s, \"\n", + " \"AI research was revived by the commercial success of expert systems, a form of AI \"\n", + " \"program that simulated the knowledge and analytical skills of human experts. By 1985, \"\n", + " \"the market for AI had reached over a billion dollars. At the same time, Japan's fifth \"\n", + " \"generation computer project inspired the U.S. and British governments to restore funding \"\n", + " \"for academic research. However, beginning with the collapse of the Lisp Machine market \"\n", + " \"in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began.\"},\n", + "\n", + " {\"query\": \"When will employment start?\",\n", + " \"answer\": \"April 16, 2012.\",\n", + " \"context\": \"THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered \"\n", + " \"into this 2nd day of April, 2012, by and between Aphrodite Apollo \"\n", + " \"(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), \"\n", + " \"and shall become effective upon Executive’s commencement of employment \"\n", + " \"(the “Effective Date”) which is expected to commence on April 16, 2012. \"\n", + " \"The Company and Executive agree that unless Executive has commenced \"\n", + " \"employment with the Company as of April 16, 2012 (or such later date as \"\n", + " \"agreed by each of the Company and Executive) this Agreement shall be \"\n", + " \"null and void and of no further effect.\"}\n", + " ]\n", + "\n", + " return test_list" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Rczsud2kOhqp" + }, + "source": [ + "Now, let's try to run these test cases with different models of Dragon and Bling series.\n", + "\n", + "We will be printing:\n", + "- LLM Response\n", + "- Gold Answer\n", + "- LLM Usage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "UpcI5QIjOzSQ" + }, + "outputs": [], + "source": [ + "def llmware_bling_dragon_hello_world (model_name):\n", + "\n", + " \"\"\" Simple inference loop that loads a model and runs through a series of test questions. \"\"\"\n", + "\n", + " t0 = time.time()\n", + " test_list = hello_world_questions()\n", + "\n", + " print(f\"\\n > Loading Model: {model_name}...\")\n", + "\n", + " # please note that by default, we recommend setting temperature=0.0 and sample=False for fact-based RAG\n", + " prompter = Prompt().load_model(model_name, temperature=0.0, sample=False)\n", + "\n", + " t1 = time.time()\n", + " print(f\"\\n > Model {model_name} load time: {t1-t0} seconds\")\n", + "\n", + " for i, entries in enumerate(test_list):\n", + " print(f\"\\n{i+1}. Query: {entries['query']}\")\n", + "\n", + " # run the prompt\n", + " output = prompter.prompt_main(entries[\"query\"],context=entries[\"context\"], prompt_name=\"default_with_context\")\n", + "\n", + " llm_response = output[\"llm_response\"].strip(\"\\n\")\n", + " print(f\"LLM Response: {llm_response}\")\n", + " print(f\"Given Answer: {entries['answer']}\")\n", + " print(f\"LLM Usage: {output['usage']}\")\n", + "\n", + " t2 = time.time()\n", + " print(f\"\\nTotal processing time: {t2-t1} seconds\")\n", + "\n", + " return 0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KALiWrOtO7N9" + }, + "source": [ + "Below is the main function with a series of models listed. Select the models one by one by just replacing the name of `my_model = bling_gguf[1]` to the one you want to use with the index i.e. the place where the model you want to test is present.\n", + "\n", + "For example, if I want to run dragon_gguf -> \"dragon-mistral-answer-tool\" model which is present at position 3 or index 2 (indexing starts from 0), therefore we will use `my_model = dragon_gguf[2]`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "tuNXtu4GPDoV" + }, + "outputs": [], + "source": [ + "if __name__ == \"__main__\":\n", + "\n", + " bling_pytorch = [\n", + "\n", + " # pytorch models - will run fast on GPU, and smaller ones good for CPU only\n", + " # note: you will need to install pytorch and transformers to pull and access these models\n", + "\n", + " \"llmware/bling-1b-0.1\",\n", + " \"llmware/bling-tiny-llama-v0\",\n", + " \"llmware/bling-1.4b-0.1\",\n", + " \"llmware/bling-falcon-1b-0.1\",\n", + " \"llmware/bling-cerebras-1.3b-0.1\",\n", + " \"llmware/bling-sheared-llama-1.3b-0.1\",\n", + " \"llmware/bling-sheared-llama-2.7b-0.1\",\n", + " \"llmware/bling-red-pajamas-3b-0.1\",\n", + " \"llmware/bling-stable-lm-3b-4e1t-v0\",\n", + " \"llmware/bling-phi-3\",\n", + " \"llmware/bling-phi-3.5\"\n", + " ]\n", + "\n", + " dragon_pytorch = [\n", + "\n", + " # pytorch models - intended for GPU server use - will require pytorch, transformers, and in some cases,\n", + " # other dependencies (einops, flash_attn).\n", + "\n", + " \"llmware/dragon-mistral-7b-v0\",\n", + " \"llmware/dragon-yi-6b-v0\",\n", + " \"llmware/dragon-qwen-7b\",\n", + " \"llmware/dragon-llama-7b-v0\",\n", + " \"llmware/dragon-mistral-0.3\",\n", + " \"llmware/dragon-llama-3.1\",\n", + " \"llmware/dragon-deci-7b-v0\"\n", + " ]\n", + "\n", + " bling_gguf = [\n", + "\n", + " # smaller cpu-oriented models - optimal for running on a CPU\n", + "\n", + " \"bling-phi-3.5-gguf\", # **NEW** - phi-3.5 (3.8b)\n", + " \"bling-answer-tool\", # this is quantized bling-tiny-llama (1.1b)\n", + " \"bling-qwen-0.5b-gguf\", # **NEW** - qwen2 (0.5b)\n", + " \"bling-qwen-1.5b-gguf\", # **NEW** - qwen2 (1.5b)\n", + " \"bling-stablelm-3b-tool\", # quantized bling-stablelm-3b (2.7b)\n", + " \"bling-phi-3-gguf\", # quantized phi-3 (3.8b)\n", + " \"bling-phi-2-gguf\", # quantized phi-2 (2.7b)\n", + " ]\n", + "\n", + " dragon_gguf = [\n", + "\n", + " # larger models - 6b - 9b\n", + "\n", + " \"dragon-yi-answer-tool\", # quantized yi-6b (v1) (6b)\n", + " \"dragon-llama-answer-tool\",\n", + " \"dragon-mistral-answer-tool\",\n", + " \"dragon-qwen-7b-gguf\", # **NEW** qwen2-7b (7b)\n", + " \"dragon-yi-9b-gguf\", # **NEW** yi-9b (8.8b)\n", + " \"dragon-llama-3.1-gguf\",\n", + " \"dragon-mistral-0.3-gguf\"\n", + "\n", + " ]\n", + "\n", + " # for most use cases, we would recommend using the GGUF for faster inference\n", + " # NEW - if you are running on a Windows machine, then try substituting for one of the following:\n", + " # -- \"bling-tiny-llama-ov\" -> uses OpenVino model version - requires `pip install openvino` and `pip install openvino_genai`\n", + " # -- \"bling-tiny-llama-onnx\" -> uses ONNX model version - requires `pip install onnxruntime_genai`\n", + "\n", + " my_model = bling_gguf[1]\n", + "\n", + " llmware_bling_dragon_hello_world(my_model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "7EM24n7gMxWw" + }, + "source": [ + "# Final Notes 🌪️\n", + "\n", + "We are always updating the BLING and DRAGON model collection with new models, including improvements in the training techniques and experimenting with new base models, and try to keep this script updated as a good 'hello world' sandbox entry point for testing and evaluation.\n", + "\n", + "We have trained these models on a wide range of base foundation models to also support preferences among specific users and clients for a particular base model.\n", + "\n", + "We score each of these models on a RAG benchmark test for accuracy and a number of specialized metrics. Please see the example \"`get_model_benchmarks`\" for a view of this.\n", + "\n", + "Loved it?? This is just an example of our models. Please check out our other Agentic AI examples with every model in detail here: https://github.com/llmware-ai/llmware/blob/main/fast_start/agents/agents-2-llmware_model_sampler_bling_dragon.py\n", + "\n", + "Also, if you have more interest in RAG, then please go with our RAG examples, which you can find here: https://github.com/llmware-ai/llmware/tree/main/fast_start/rag\n", + "\n", + "If you liked it, then please **star our repo** https://github.com/llmware-ai/llmware ⭐\n", + "\n", + "Any doubts?? Join our **discord server: https://discord.gg/GN49aWx2H3** 🫂" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/Agents_03.ipynb b/tutorials/notebooks/Agents_03.ipynb new file mode 100644 index 0000000..a2f86d0 --- /dev/null +++ b/tutorials/notebooks/Agents_03.ipynb @@ -0,0 +1,294 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Welcome to Slim Extract Fast Start by LLMWare.ai 🦾\n", + "This is the 3rd tutorial of our Agents Fast Start Series. In this tutorial, we will dive into the use of Sliim Extract model.\n", + "\n", + "\n", + "## Slim Extract Model ...\n", + "\n", + "SLIMs are **Structured Language Instruction Models**, which are small, specialized 1-3B parameter LLMs, finetuned to generate structured outputs (Python dictionaries and lists, JSON and SQL) that can be handled programmatically, and stacked together in multi-step, multi-model Agent workflows - all running on a local CPU.\n", + "\n", + "The Slim-extract model are used to **extract custom keys from selected text**.\n" + ], + "metadata": { + "id": "7nv_BZnr7MUU" + } + }, + { + "cell_type": "code", + "source": [ + "# install dependencies\n", + "!pip3 install llmware # if you're getting error, then try upgrading the pip by runnnig this command : python -m pip install --upgrade pip" + ], + "metadata": { + "id": "K68hEcgu78my" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "If you have any dependency install issues, please review the README, docs link, or raise an Issue.\n", + "\n", + "Usually, if there is a missing dependency, the code will give the warning - and a clear direction like `pip install transformers'` required for this example, etc.\n", + "\n", + "As an alternative to pip install ... if you prefer, you can also clone the repo from github which provides a benefit of having access to 100+ examples.\n", + "\n", + "To clone the repo:\n", + "```\n", + "git clone \"https://www.github.com/llmware-ai/llmware.git\"\n", + "sh \"welcome_to_llmware.sh\"\n", + "```\n", + "\n", + "The second script `\"welcome_to_llmware.sh\"` will install all of the dependencies.\n", + "\n", + "If using Windows, then use the `\"welcome_to_llmware_windows.sh\"` script." + ], + "metadata": { + "id": "TjpSZbx179pB" + } + }, + { + "cell_type": "code", + "source": [ + "# Import the ModelCatalog\n", + "from llmware.models import ModelCatalog" + ], + "metadata": { + "id": "DFpu9BOd8Ax6" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "We have included a set of sample earnings releases (comprising lines ~10 - ~385 of this script), and run a simple loop through the earnings releases, showing how to create an extract prompt to identify 'revenue growth' from these examples.\n", + "\n", + "here are several function-calling models in the slim-extract family, fine-tuned on multiple leading small model base foundations - full list and options are below in the code." + ], + "metadata": { + "id": "7U3N9zlV8PkI" + } + }, + { + "cell_type": "code", + "source": [ + "earnings_releases = [\n", + "\n", + " {\"context\": \"Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker \"\n", + " \"issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. \"\n", + " \"Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: \"\n", + " \"Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected \"\n", + " \"Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. \"\n", + " \"Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, \"\n", + " \"in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of \"\n", + " \"design software startup Figma after U.K. regulators found competitive concerns. The company paid \"\n", + " \"Figma a $1 billion termination fee.\"},\n", + "\n", + " {\"context\": \"Dick’s Sporting Goods raised its dividend by 10% on Thursday as the company posted its largest sales \"\n", + " \"quarter in its history and projected another year of growth. The company’s shares jumped more than \"\n", + " \"15% in intraday trading. CEO Lauren Hobart said on an earnings call Thursday that Dick’s sales \"\n", + " \"growth came from bigger tickets — either higher prices or more expensive items — as its transactions \"\n", + " \"were flat. Many retailers benefited from a 53rd week in fiscal 2023, but Dick’s said it still broke \"\n", + " \"records during its fiscal fourth quarter even without those extra days. Here’s how the athletic \"\n", + " \"apparel retailer did compared with what Wall Street was anticipating, based on a survey of \"\n", + " \"analysts by LSEG, formerly known as Refinitiv: Earnings per share: $3.85 adjusted vs. $3.35 expected \"\n", + " \"Revenue: $3.88 billion vs. $3.80 billion expected The company’s reported net income for the three-month \"\n", + " \"period that ended Feb. 3 was $296 million, or $3.57 per share, compared with $236 million, or $2.60 a \"\n", + " \"share, a year earlier. Excluding one-time items related to impairment charges and inventory write-offs, \"\n", + " \"Dick’s reported earnings per share of $3.85. Sales rose to $3.88 billion, up about 8% from $3.60 billion \"\n", + " \"a year earlier. “With our industry-leading assortment and strong execution, we capped off the year \"\n", + " \"with an incredibly strong fourth quarter and holiday season,” Hobart said in a statement. “We are \"\n", + " \"guiding to another strong year in 2024. We plan to grow both our sales and earnings through \"\n", + " \"positive comps, higher merchandise margin and productivity gains,” she added. During the quarter, \"\n", + " \"same-store sales rose 2.8%, well ahead of the 0.8% lift that analysts had expected, according to \"\n", + " \"StreetAccount. “Growth in transactions” and market share gains drove the increase, said Executive \"\n", + " \"Chairman Ed Stack.\"},\n", + "\n", + " {\"context\": \"Comcast topped both revenue and profit estimates in the fourth quarter as it lost fewer broadband \"\n", + " \"subscribers than expected, and it raised its dividend 7%, the company said Thursday. \"\n", + " \"Here’s how Comcast performed, compared with estimates from analysts surveyed by LSEG, \"\n", + " \"formerly known as Refinitiv. Earnings per share: 84 cents adjusted vs. 79 cents expected \"\n", + " \"Revenue: $31.25 billion vs. $30.51 billion expected For the quarter ended Dec. 31, net \"\n", + " \"income rose 7.8% to $3.26 billion, or 81 cents a share, compared with $3.02 billion, or \"\n", + " \"70 cents a share, a year earlier. Revenue increased 2.3% compared with the prior-year period. \"\n", + " \"Adjusted earnings before interest, taxes, depreciation and amortization (EBITDA) was flat year \"\n", + " \"over year at about $8 billion. 'For the third consecutive year, we generated the highest revenue, \"\n", + " \"adjusted EBITDA and adjusted EPS in our company’s history', Comcast Chief Executive Officer Brian \"\n", + " \"Roberts said in a statement. 'We also reported the highest adjusted EBITDA on record at Theme Parks; \"\n", + " \"were the #1 studio in worldwide box office for the first time since 2015; and maintained Peacock’s \"\n", + " \"position as the fastest growing streamer in the U.S.'\"},\n", + "\n", + " {\"context\": \"Dollar General forecast annual sales above Wall Street estimates on Thursday, banking on higher \"\n", + " \"demand from inflation-hit customers buying groceries and essentials from the discount retailer’s stores. \"\n", + " \"Shares of the company rose about 6% in early trading, after falling nearly 45% in 2023 on rising costs \"\n", + " \"and stiff competition from bigger retailers. But higher prices and borrowing costs have prompted \"\n", + " \"budget-conscious consumers to cook more meals at home, helping Dollar General record stronger \"\n", + " \"footfall at its outlets as shoppers hunt for lower-margin, needs-based goods, over pricier general \"\n", + " \"merchandise. “With customer traffic growth and market share gains during the quarter, we believe our \"\n", + " \"actions are resonating with customers,” CEO Todd Vasos said in a statement. Vasos’s strategy - to focus \"\n", + " \"on the basics, like more employee presence at stores, greater customer engagement and expanding \"\n", + " \"private-label brands - has helped stabilize Dollar General’s business. Over the last few quarters, \"\n", + " \"Dollar General and rival Dollar Tree have struggled with rising costs linked to their supply \"\n", + " \"chains, labor and raw materials, while facing tough competition from retailers like Walmart \"\n", + " \"and Chinese ecommerce platform Temu. Dollar Tree’s shares fell more than 15% on Wednesday, after it \"\n", + " \"forecast weak sales and profit for 2024 and laid out plans to shutter 970 of its Family Dollar \"\n", + " \"stores. “Dollar General has a much rosier outlook than Dollar Tree... Dollar Tree’s challenges \"\n", + " \"with Family Dollar were years in the making, while Dollar General has embarked on an aggressive \"\n", + " \"effort to add more frozen, refrigerated and fresh produce,” eMarketer senior analyst Zak Stambor said. \"\n", + " \"Dollar General forecast 2024 sales to grow between 6.0% and 6.7%, above analysts’ estimate of 4.4% \"\n", + " \"growth to $40.33 billion, according to LSEG data. It still sees annual per-share profit between \"\n", + " \"$6.80 and $7.55, compared with estimates of $7.55. Its fourth-quarter net sales of $9.86 billion \"\n", + " \"surpassed estimates of $9.78 billion. It also reported an estimate-beating profit of $1.83 per share.\"},\n", + "\n", + " {\"context\": \"Best Buy surpassed Wall Street’s revenue and earnings expectations for the holiday quarter on \"\n", + " \"Thursday, even as the company navigated through a period of tepid consumer electronics demand. \"\n", + " \"But the retailer warned of another year of softer sales and said it would lay off workers and \"\n", + " \"cut other costs across the business. CEO Corie Barry offered few specifics, but said the \"\n", + " \"company has to make sure its workforce and stores match customers’ changing shopping habits. \"\n", + " \"Cuts will free up capital to invest back into the business and in newer areas, such as artificial \"\n", + " \"intelligence, she added. “This is giving us some of that space to be able to reinvest into \"\n", + " \"our future and make sure we feel like we are really well positioned for the industry to \"\n", + " \"start to rebound,” she said on a call with reporters. For this fiscal year, Best Buy anticipates \"\n", + " \"revenue will range from $41.3 billion to $42.6 billion. That would mark a drop from the most \"\n", + " \"recently ended fiscal year, when full-year revenue totaled $43.45 billion. It said comparable \"\n", + " \"sales will range from flat to a 3% decline. The retailer plans to close 10 to 15 stores \"\n", + " \"this year after shuttering 24 in the past fiscal year. One challenge that will affect sales \"\n", + " \"in the year ahead: it is a week shorter. Best Buy said the extra week in the past fiscal \"\n", + " \"year lifted revenue by about $735 million and boosted diluted earnings per share by about \"\n", + " \"30 cents. Shares of Best Buy closed more than 1% higher Thursday after briefly touching \"\n", + " \"a 52-week high of $86.11 earlier in the session. Here’s what the consumer electronics \"\n", + " \"retailer reported for its fiscal fourth quarter of 2024 compared with what Wall Street was \"\n", + " \"expecting, based on a survey of analysts by LSEG, formerly known as Refinitiv: \"\n", + " \"Earnings per share: $2.72, adjusted vs. $2.52 expected Revenue: $14.65 billion vs. $14.56 \"\n", + " \"billion expected A dip in demand, but a better-than-feared holiday Best Buy has dealt \"\n", + " \"with slower demand in part due to the strength of its sales during the pandemic. Like \"\n", + " \"home improvement companies, Best Buy saw outsized spending as shoppers were stuck at \"\n", + " \"home. Plus, many items that the retailer sells like laptops, refrigerators and home \"\n", + " \"theater systems tend to be pricier and less frequent purchases. The retailer has cited other \"\n", + " \"challenges, too: Shoppers have been choosier about making big purchases while dealing \"\n", + " \"with inflation-driven higher prices of food and more. Plus, they’ve returned to \"\n", + " \"splitting their dollars between services and goods after pandemic years of little \"\n", + " \"activity. Even so, Best Buy put up a holiday quarter that was better than feared. \"\n", + " \"In the three-month period that ended Feb. 3, the company’s net income fell by 7% to \"\n", + " \"$460 million, or $2.12 per share, from $495 million, or $2.23 per share in the year-ago \"\n", + " \"period. Revenue dropped from $14.74 billion a year earlier. Comparable sales, a metric that \"\n", + " \"includes sales online and at stores open at least 14 months, declined 4.8% during the \"\n", + " \"quarter as shoppers bought fewer appliances, mobile phones, tablets and home theater \"\n", + " \"setups than the year-ago period. Gaming, on the other hand, was a strong sales \"\n", + " \"category in the holiday quarter.\"}\n", + "\n", + "]" + ], + "metadata": { + "id": "I49eI2UI8RaX" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Available Slim Extract Models\n", + "slim_extract_models = [\"slim-extract-tool\", # original - stablelm-3b (2.7b)\n", + " \"slim-extract-tiny-tool\", # tiny-llama 1.1b\n", + " \"slim-extract-qwen-1.5b-gguf\", # **NEW** qwen 1.5b\n", + " \"slim-extract-phi-3-gguf\", # **NEW** phi-3 (3.8b)\n", + " \"slim-extract-qwen-0.5b-gguf\"] # **NEW** qwen 0.5b" + ], + "metadata": { + "id": "wbtHQKMP8rDY" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# Load the Model\n", + "model = ModelCatalog().load_model(\"slim-extract-tool\",sample=False,temperature=0.0, max_output=100) # You can change the model from any of the above one and test it yourself." + ], + "metadata": { + "id": "5STiKpjk83_s" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# iterate through the earnings release samples above\n", + "for i, sample in enumerate(earnings_releases):\n", + "\n", + " # key line: execute function_call on selected model with 'custom_key' = \"revenue growth\"\n", + " response = model.function_call(sample[\"context\"], function=\"extract\", params=[\"revenue growth\"])\n", + "\n", + " # display the response on the screen\n", + " print(\"extract response: \", i, response[\"llm_response\"])" + ], + "metadata": { + "id": "SW_J08CB88LC" + }, + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "# Final Notes 🐛\n", + "SLIM or Structured Language Instruction Models are powerful small models and have a great use case.\n", + "\n", + "In this fast start series, we experimented with the SLIM-EXTRACT MODEL that has a great use case in extracting custom keys from selected text.\n", + "\n", + "The available SLIM EXTRACT MODELS are:\n", + "- slim-extract-tool [original - stablelm-3b (2.7b)]\n", + "- slim-extract-tiny-tool [tiny-llama 1.1b]\n", + "- slim-extract-qwen-1.5b-gguf [**NEW** qwen 1.5b]\n", + "- slim-extract-phi-3-gguf [**NEW** phi-3 (3.8b)]\n", + "- slim-extract-qwen-0.5b-gguf [**NEW** qwen 0.5b]\n", + "\n", + "HomeWork:\n", + "Test all these on your own with your cutsom test cases!" + ], + "metadata": { + "id": "lXq7-toh9jwR" + } + }, + { + "cell_type": "markdown", + "source": [ + "## Journey is yet to start!\n", + "Loved it?? This is just an example of our models. Please check out our other Agentic AI examples with every model in detail here: https://github.com/llmware-ai/llmware/tree/main/fast_start/agents\n", + "\n", + "Also, if you have more interest in RAG, then please go with our RAG examples, which you can find here: https://github.com/llmware-ai/llmware/tree/main/fast_start/rag\n", + "\n", + "If you liked it, then please **star our repo https://github.com/llmware-ai/llmware** ⭐\n", + "\n", + "Any doubts?? Join our **discord server: https://discord.gg/GN49aWx2H3** 🫂" + ], + "metadata": { + "id": "HoVSbPiX9bwG" + } + } + ], + "metadata": { + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git a/tutorials/notebooks/NoteBook_Examples/agent_multistep_analysis_nb.ipynb b/tutorials/notebooks/NoteBook_Examples/agent_multistep_analysis_nb.ipynb new file mode 100644 index 0000000..b7bb7cb --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/agent_multistep_analysis_nb.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "6sr3Gk8hNDaF" + }, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eQk5Dv3jyheg" + }, + "source": [ + "# If you are using Colab for free, we highly recommend you activate the T4 GPU\n", + "# hardware accelerator. Our models are designed to run with at least 16GB\n", + "# of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed\n", + "# to the ~13GB Colab gives automatically.\n", + "# To activate T4:\n", + "# 1. click on the \"Runtime\" tab\n", + "# 2. click on \"Change runtime type\"\n", + "# 3. select T4 GPU under Hardware Accelerator\n", + "# NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "td7dVt4fD_j6" + }, + "outputs": [], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "IY8ysw9FElZR" + }, + "outputs": [], + "source": [ + "!pip install --upgrade huggingface_hub\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "AeR-LvVXD27i" + }, + "outputs": [], + "source": [ + "\n", + "\"\"\" This example shows a complex multi-part research analysis. In this example, we will:\n", + "\n", + " 1. Build a \"research\" library.\n", + " 2. Query the research library to identify topics of interest.\n", + " 3. Create an agent with several analytical tools: sentiment, emotions, topic, entities analysis\n", + " 4. Pass the results of our query to the agent to conduct multifaceted analysis.\n", + " 5. Apply a top-level filter ('sentiment') on the results from the query\n", + " 6. For any of the passages with negative sentiment, we will run a follow-up set of analyses.\n", + " 7. Finally, we will assemble the follow-up analysis into a list of detailed reports.\n", + "\"\"\"\n", + "\n", + "import os\n", + "import shutil\n", + "\n", + "from llmware.agents import LLMfx\n", + "from llmware.library import Library\n", + "from llmware.retrieval import Query\n", + "from llmware.configs import LLMWareConfig\n", + "from llmware.setup import Setup\n", + "\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "qG3lAj2nM372" + }, + "outputs": [], + "source": [ + "def multistep_analysis():\n", + "\n", + " \"\"\" In this example, our objective is to research Microsoft history and rivalry in the 1980s with IBM. \"\"\"\n", + "\n", + " # step 1 - assemble source documents and create library\n", + "\n", + " print(\"update: Starting example - agent-multistep-analysis\")\n", + "\n", + " # note: lines 38-49 attempt to automatically pull sample document into local path\n", + " # depending upon permissions in your environment, you may need to set up directly\n", + " # if you pull down the samples files with Setup().load_sample_files(), in the Books folder,\n", + " # you will find the source: \"Bill-Gates-Biography.pdf\"\n", + " # if you have pulled sample documents in the past, then to update to latest: set over_write=True\n", + "\n", + " print(\"update: Loading sample files\")\n", + "\n", + " sample_files_path = Setup().load_sample_files(over_write=False)\n", + " bill_gates_bio = \"Bill-Gates-Biography.pdf\"\n", + " path_to_bill_gates_bio = os.path.join(sample_files_path, \"Books\", bill_gates_bio)\n", + "\n", + " microsoft_folder = os.path.join(LLMWareConfig().get_tmp_path(), \"example_microsoft\")\n", + "\n", + " print(\"update: attempting to create source input folder at path: \", microsoft_folder)\n", + "\n", + " if not os.path.exists(microsoft_folder):\n", + " os.mkdir(microsoft_folder)\n", + " os.chmod(microsoft_folder, 0o777)\n", + " shutil.copy(path_to_bill_gates_bio,os.path.join(microsoft_folder, bill_gates_bio))\n", + "\n", + " # create library\n", + " print(\"update: creating library and parsing source document\")\n", + "\n", + " LLMWareConfig().set_active_db(\"sqlite\")\n", + " my_lib = Library().create_new_library(\"microsoft_history_0210_1\")\n", + " my_lib.add_files(microsoft_folder)\n", + "\n", + " # run our first query - \"ibm\"\n", + " query = \"ibm\"\n", + " search_results = Query(my_lib).text_query(query)\n", + " print(f\"update: executing query to filter to key passages - {query} - results found - {len(search_results)}\")\n", + "\n", + " # create an agent and load several tools that we will be using\n", + " agent = LLMfx()\n", + " agent.load_tool_list([\"sentiment\", \"emotions\", \"topic\", \"tags\", \"ner\", \"answer\"])\n", + "\n", + " # load the search results into the agent's work queue\n", + " agent.load_work(search_results)\n", + "\n", + " while True:\n", + "\n", + " agent.sentiment()\n", + "\n", + " if not agent.increment_work_iteration():\n", + " break\n", + "\n", + " # analyze sections where the sentiment on ibm was negative\n", + " follow_up_list = agent.follow_up_list(key=\"sentiment\", value=\"negative\")\n", + "\n", + " for job_index in follow_up_list:\n", + "\n", + " # follow-up 'deep dive' on selected text that references ibm negatively\n", + " agent.set_work_iteration(job_index)\n", + " agent.exec_multitool_function_call([\"tags\", \"emotions\", \"topics\", \"ner\"])\n", + " agent.answer(\"What is a brief summary?\", key=\"summary\")\n", + "\n", + " my_report = agent.show_report(follow_up_list)\n", + "\n", + " activity_summary = agent.activity_summary()\n", + "\n", + " for entries in my_report:\n", + " print(\"my report entries: \", entries)\n", + "\n", + " return my_report" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NDKDVuEoM9rL" + }, + "outputs": [], + "source": [ + "\n", + "if __name__ == \"__main__\":\n", + "\n", + " multistep_analysis()\n", + "\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/NoteBook_Examples/agent_with_custom_tables_example.ipynb b/tutorials/notebooks/NoteBook_Examples/agent_with_custom_tables_example.ipynb new file mode 100644 index 0000000..cf2f357 --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/agent_with_custom_tables_example.ipynb @@ -0,0 +1,468 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "eQk5Dv3jyheg" + }, + "source": [ + "# If you are using Colab for free, we highly recommend you activate the T4 GPU\n", + "# hardware accelerator. Our models are designed to run with at least 16GB\n", + "# of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed\n", + "# to the ~13GB Colab gives automatically.\n", + "# To activate T4:\n", + "# 1. click on the \"Runtime\" tab\n", + "# 2. click on \"Change runtime type\"\n", + "# 3. select T4 GPU under Hardware Accelerator\n", + "# NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "POFehtFWxwlz" + }, + "source": [ + "\n", + " This example shows an end-to-end recipe for creating a CustomTable, and then creating an Agent process that\n", + " will query the table using natural language.\n", + "\n", + " Please note that this example is a 'generalized' and updated version of an earlier example -\n", + " \"text2sql-end-to-end-2.py\" - now using the more powerful CustomTables class integrated into the LLMfx process\n", + "\n", + " The example shows the following steps:\n", + "\n", + " 1. Creating a custom table resource from a sample CSV file, included in the slim-sql-tool kit, and also\n", + " available in the Examples section with Structured_Tables (customer_table.csv)\n", + "\n", + " 2 Asking basic natural language questions:\n", + " A. Looks up the table schema\n", + " B. Packages the table schema with query\n", + " C. Runs inference to convert text into SQL\n", + " D. Queries the database with the generated SQL\n", + " E. Returns result\n", + "\n", + " 3. Using CustomtTable class, this can be run on either Postgres or SQLite DB.\n", + "\n", + " Note: as you substitute for your own CSV and JSON, check out the other examples in this section for loading\n", + " configuration ideas and options.\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gWyhnhW4v9zm" + }, + "source": [ + "This notebook is designed for the google collab environment\n", + "\n", + "\n", + "1) The first cell will allow this notebook to access files in your google drive. The example csv for this notebook is located at https://github.com/llmware-ai/llmware/blob/main/examples/Structured_Tables/customer_table.csv\n", + "\n", + "Upload it to your drive, and ensure that when you input parameters you point to the right path where the csv is located.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "AFH-qyZ_tVpn", + "outputId": "1a51d6ea-af94-441c-c99b-e2457108e8dc" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n" + ] + } + ], + "source": [ + "from google.colab import drive\n", + "drive.mount('/content/drive')\n" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "sDjd_1IPoGx9", + "outputId": "575d6053-98b6-4098-defb-e1c3cce6777b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: llmware in /usr/local/lib/python3.10/dist-packages (0.3.0)\n", + "Requirement already satisfied: boto3>=1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.34.124)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.7.3)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.124 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.34.124)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (0.10.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.124->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.124->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.124->boto3>=1.24.53->llmware) (1.16.0)\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "id": "zAUBmgSptRgQ" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.agents import LLMfx\n", + "from llmware.resources import CustomTable\n", + "from llmware.configs import LLMWareConfig\n" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "id": "P0okOaH0taa0" + }, + "outputs": [], + "source": [ + "def build_table(db=None, table_name=None, load_fp=None, load_file=None):\n", + " \"\"\"Simple example script to take a CSV or JSON/JSONL and create a DB Table.\"\"\"\n", + " custom_table = CustomTable(db=db, table_name=table_name)\n", + " analysis = custom_table.validate_csv(load_fp, load_file)\n", + " print(\"update: analysis from validate_csv: \", analysis)\n", + "\n", + " if load_file.endswith(\".csv\"):\n", + " output = custom_table.load_csv(load_fp, load_file)\n", + " elif load_file.endswith(\".jsonl\") or load_file.endswith(\".json\"):\n", + " output = custom_table.load_json(load_fp, load_file)\n", + " else:\n", + " print(\"file type not supported for db load\")\n", + " return -1\n", + "\n", + " print(\"update: output from loading file: \", output)\n", + "\n", + " sample_range = min(10, len(custom_table.rows))\n", + " for x in range(0, sample_range):\n", + " print(\"update: sample rows: \", x, custom_table.rows[x])\n", + "\n", + " updated_schema = custom_table.test_and_remediate_schema(samples=20, auto_remediate=True)\n", + " print(\"update: updated schema: \", updated_schema)\n", + "\n", + " custom_table.insert_rows()\n", + " return 1\n" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "id": "Q6uD3D-wtfOG" + }, + "outputs": [], + "source": [ + "def agent_natural_language_sql_query(query_list, db=None, table_name=None):\n", + " \"\"\"Query a CustomTable in natural language.\"\"\"\n", + " agent = LLMfx()\n", + " agent.load_tool(\"sql\", sample=False, get_logits=True, temperature=0.0)\n", + "\n", + " for i, query in enumerate(query_list):\n", + " response = agent.query_custom_table(query, db=db, table=table_name)\n", + "\n", + " for x in range(len(agent.research_list)):\n", + " print(\"research: \", x, agent.research_list[x])\n", + "\n", + " return agent.research_list\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "bJaDKQyAthUO", + "outputId": "1931b424-b49f-4176-8c0f-aa2c68d5304a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: analysis from validate_csv: {'rows': 20, 'columns': 6, 'conforming_rows_percent': 1.0, 'column_frequency_analysis': {6: 20}, 'nonconforming_rows': []}\n", + "update: output from loading file: {'rows': 19, 'columns': 6, 'schema': {'customer_name': 'text', 'account_number': 'integer', 'customer_level': 'text', 'vip_customer': 'text', 'annual_spend': 'integer', 'user_name': 'text'}, 'skipped_rows': []}\n", + "update: sample rows: 0 {'customer_name': 'Martha Williams', 'account_number': '98320893', 'customer_level': 'gold', 'vip_customer': 'yes', 'annual_spend': '63250', 'user_name': 'mwilliams'}\n", + "update: sample rows: 1 {'customer_name': 'Susan Soinsin', 'account_number': '53439382', 'customer_level': 'silver', 'vip_customer': 'no', 'annual_spend': '112 ', 'user_name': 'ssinsin'}\n", + "update: sample rows: 2 {'customer_name': 'Michael Rogers', 'account_number': '88888444', 'customer_level': 'bronze', 'vip_customer': 'no', 'annual_spend': '3 ', 'user_name': 'rogersm'}\n", + "update: sample rows: 3 {'customer_name': 'John Jones', 'account_number': '9898482', 'customer_level': 'gold', 'vip_customer': 'yes', 'annual_spend': '12430', 'user_name': 'jjns'}\n", + "update: sample rows: 4 {'customer_name': 'Melinda Lyons', 'account_number': '1234953', 'customer_level': 'silver', 'vip_customer': 'no', 'annual_spend': '234', 'user_name': 'mlyons'}\n", + "update: sample rows: 5 {'customer_name': 'Arvind Arora', 'account_number': '8998899', 'customer_level': 'gold', 'vip_customer': 'yes', 'annual_spend': '42650', 'user_name': 'aarora'}\n", + "update: sample rows: 6 {'customer_name': 'Paul Rinno', 'account_number': '3829235', 'customer_level': 'silver', 'vip_customer': 'no', 'annual_spend': '1293', 'user_name': 'prinno'}\n", + "update: sample rows: 7 {'customer_name': 'Vinod Aggarwal', 'account_number': '9389532', 'customer_level': 'platinum', 'vip_customer': 'yes', 'annual_spend': '93540', 'user_name': 'aggarwal'}\n", + "update: sample rows: 8 {'customer_name': 'Bryan Lewis', 'account_number': '12399853', 'customer_level': 'gold', 'vip_customer': 'no', 'annual_spend': '9732', 'user_name': 'brlewis'}\n", + "update: sample rows: 9 {'customer_name': 'Allison Winters', 'account_number': '83902867', 'customer_level': 'gold', 'vip_customer': 'yes', 'annual_spend': '15000', 'user_name': 'wintersall'}\n", + "update: updated schema: {'customer_name': 'text', 'account_number': 'integer', 'customer_level': 'text', 'vip_customer': 'text', 'annual_spend': 'integer', 'user_name': 'text'}\n" + ] + }, + { + "data": { + "text/plain": [ + "1" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Input parameters\n", + "db = \"sqlite\" # SQLite database path\n", + "table_name = \"customer_table\"\n", + "\n", + "# Path to the CSV file in Google Drive\n", + "input_fp = \"/content/drive/My Drive\"\n", + "input_fn = \"customer_table.csv\"\n", + "\n", + "# Build the table\n", + "build_table(db=db, table_name=table_name, load_fp=input_fp, load_file=input_fn)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "FJQlgHC9uq8Q", + "outputId": "2a9f6715-2cd7-41b5-aca6-ed3c9f05fb0e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Launching LLMfx process\n", + "step - \t1 - \tcreating object - ready to start processing.\n", + "step - \t2 - \tloading tool - sql\n", + "step - \t3 - \tloading new processing text - 1 new entries\n", + "step - \t4 - \texecuting function call - deploying - text-to-sql\n", + "\t\t\t\t -- query - Which customers have vip customer status of yes?\n", + "\t\t\t\t -- table_schema - CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )\n", + "step - \t5 - \texecuting function call - getting response - sql\n", + "\t\t\t\t -- llm_response - SELECT customer_name FROM customer_table WHERE vip_customer = 'yes'\n", + "\t\t\t\t -- output type - text\n", + "\t\t\t\t -- usage - {'input': 58, 'output': 17, 'total': 75, 'metric': 'tokens', 'processing_time': 1.0267748832702637}\n", + "step - \t6 - \texecuting research call - executing query on db\n", + "\t\t\t\t -- db - sqlite\n", + "\t\t\t\t -- sql_query - SELECT customer_name FROM customer_table WHERE vip_customer = 'yes'\n", + "step - \t7 - \texecuting research - getting response - sql\n", + "\t\t\t\t -- result - [('Martha Williams',), ('John Jones',), ('Arvind Arora',), ('Vinod Aggarwal',), ('Allison Winters',), ('Olivia Smith',), ('Alan Wang',), ('Wilmer Rodriguez',), ('Eliza Listron',), ('Douglas Hudson',), ('Martha Williams',), ('John Jones',), ('Arvind Arora',), ('Vinod Aggarwal',), ('Allison Winters',), ('Olivia Smith',), ('Alan Wang',), ('Wilmer Rodriguez',), ('Eliza Listron',), ('Douglas Hudson',)]\n", + "step - \t8 - \tloading new processing text - 1 new entries\n", + "step - \t9 - \texecuting function call - deploying - text-to-sql\n", + "\t\t\t\t -- query - What is the highest annual spend of any customer?\n", + "\t\t\t\t -- table_schema - CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )\n", + "step - \t10 - \texecuting function call - getting response - sql\n", + "\t\t\t\t -- llm_response - SELECT MAX(annual_spend) FROM customer_table\n", + "\t\t\t\t -- output type - text\n", + "\t\t\t\t -- usage - {'input': 57, 'output': 13, 'total': 70, 'metric': 'tokens', 'processing_time': 1.495262861251831}\n", + "step - \t11 - \texecuting research call - executing query on db\n", + "\t\t\t\t -- db - sqlite\n", + "\t\t\t\t -- sql_query - SELECT MAX(annual_spend) FROM customer_table\n", + "step - \t12 - \texecuting research - getting response - sql\n", + "\t\t\t\t -- result - [(93540,)]\n", + "step - \t13 - \tloading new processing text - 1 new entries\n", + "step - \t14 - \texecuting function call - deploying - text-to-sql\n", + "\t\t\t\t -- query - Which customer has account number 1234953\n", + "\t\t\t\t -- table_schema - CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )\n", + "step - \t15 - \texecuting function call - getting response - sql\n", + "\t\t\t\t -- llm_response - SELECT customer_name FROM customer_table WHERE account_number = 1234953\n", + "\t\t\t\t -- output type - text\n", + "\t\t\t\t -- usage - {'input': 61, 'output': 21, 'total': 82, 'metric': 'tokens', 'processing_time': 2.2324819564819336}\n", + "step - \t16 - \texecuting research call - executing query on db\n", + "\t\t\t\t -- db - sqlite\n", + "\t\t\t\t -- sql_query - SELECT customer_name FROM customer_table WHERE account_number = 1234953\n", + "step - \t17 - \texecuting research - getting response - sql\n", + "\t\t\t\t -- result - [('Melinda Lyons',), ('Melinda Lyons',)]\n", + "step - \t18 - \tloading new processing text - 1 new entries\n", + "step - \t19 - \texecuting function call - deploying - text-to-sql\n", + "\t\t\t\t -- query - Which customer has the lowest annual spend?\n", + "\t\t\t\t -- table_schema - CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )\n", + "step - \t20 - \texecuting function call - getting response - sql\n", + "\t\t\t\t -- llm_response - SELECT customer_name FROM customer_table ORDER BY annual_spend LIMIT 1\n", + "\t\t\t\t -- output type - text\n", + "\t\t\t\t -- usage - {'input': 56, 'output': 17, 'total': 73, 'metric': 'tokens', 'processing_time': 1.4454705715179443}\n", + "step - \t21 - \texecuting research call - executing query on db\n", + "\t\t\t\t -- db - sqlite\n", + "\t\t\t\t -- sql_query - SELECT customer_name FROM customer_table ORDER BY annual_spend LIMIT 1\n", + "step - \t22 - \texecuting research - getting response - sql\n", + "\t\t\t\t -- result - [('Michael Rogers',)]\n", + "step - \t23 - \tloading new processing text - 1 new entries\n", + "step - \t24 - \texecuting function call - deploying - text-to-sql\n", + "\t\t\t\t -- query - Is Susan Soinsin a vip customer?\n", + "\t\t\t\t -- table_schema - CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )\n", + "step - \t25 - \texecuting function call - getting response - sql\n", + "\t\t\t\t -- llm_response - SELECT vip_customer FROM customer_table WHERE customer_name = 'Susan Soinsin'\n", + "\t\t\t\t -- output type - text\n", + "\t\t\t\t -- usage - {'input': 57, 'output': 22, 'total': 79, 'metric': 'tokens', 'processing_time': 1.4810450077056885}\n", + "step - \t26 - \texecuting research call - executing query on db\n", + "\t\t\t\t -- db - sqlite\n", + "\t\t\t\t -- sql_query - SELECT vip_customer FROM customer_table WHERE customer_name = 'Susan Soinsin'\n", + "step - \t27 - \texecuting research - getting response - sql\n", + "\t\t\t\t -- result - [('no',), ('no',)]\n", + "research: 0 {'step': 6, 'tool': 'sql', 'db_response': [('Martha Williams',), ('John Jones',), ('Arvind Arora',), ('Vinod Aggarwal',), ('Allison Winters',), ('Olivia Smith',), ('Alan Wang',), ('Wilmer Rodriguez',), ('Eliza Listron',), ('Douglas Hudson',), ('Martha Williams',), ('John Jones',), ('Arvind Arora',), ('Vinod Aggarwal',), ('Allison Winters',), ('Olivia Smith',), ('Alan Wang',), ('Wilmer Rodriguez',), ('Eliza Listron',), ('Douglas Hudson',)], 'sql_query': \"SELECT customer_name FROM customer_table WHERE vip_customer = 'yes'\", 'query': 'Which customers have vip customer status of yes?', 'db': 'sqlite', 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'}\n", + "research: 1 {'step': 11, 'tool': 'sql', 'db_response': [(93540,)], 'sql_query': 'SELECT MAX(annual_spend) FROM customer_table', 'query': 'What is the highest annual spend of any customer?', 'db': 'sqlite', 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'}\n", + "research: 2 {'step': 16, 'tool': 'sql', 'db_response': [('Melinda Lyons',), ('Melinda Lyons',)], 'sql_query': 'SELECT customer_name FROM customer_table WHERE account_number = 1234953', 'query': 'Which customer has account number 1234953', 'db': 'sqlite', 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'}\n", + "research: 3 {'step': 21, 'tool': 'sql', 'db_response': [('Michael Rogers',)], 'sql_query': 'SELECT customer_name FROM customer_table ORDER BY annual_spend LIMIT 1', 'query': 'Which customer has the lowest annual spend?', 'db': 'sqlite', 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'}\n", + "research: 4 {'step': 26, 'tool': 'sql', 'db_response': [('no',), ('no',)], 'sql_query': \"SELECT vip_customer FROM customer_table WHERE customer_name = 'Susan Soinsin'\", 'query': 'Is Susan Soinsin a vip customer?', 'db': 'sqlite', 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'}\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'step': 6,\n", + " 'tool': 'sql',\n", + " 'db_response': [('Martha Williams',),\n", + " ('John Jones',),\n", + " ('Arvind Arora',),\n", + " ('Vinod Aggarwal',),\n", + " ('Allison Winters',),\n", + " ('Olivia Smith',),\n", + " ('Alan Wang',),\n", + " ('Wilmer Rodriguez',),\n", + " ('Eliza Listron',),\n", + " ('Douglas Hudson',),\n", + " ('Martha Williams',),\n", + " ('John Jones',),\n", + " ('Arvind Arora',),\n", + " ('Vinod Aggarwal',),\n", + " ('Allison Winters',),\n", + " ('Olivia Smith',),\n", + " ('Alan Wang',),\n", + " ('Wilmer Rodriguez',),\n", + " ('Eliza Listron',),\n", + " ('Douglas Hudson',)],\n", + " 'sql_query': \"SELECT customer_name FROM customer_table WHERE vip_customer = 'yes'\",\n", + " 'query': 'Which customers have vip customer status of yes?',\n", + " 'db': 'sqlite',\n", + " 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'},\n", + " {'step': 11,\n", + " 'tool': 'sql',\n", + " 'db_response': [(93540,)],\n", + " 'sql_query': 'SELECT MAX(annual_spend) FROM customer_table',\n", + " 'query': 'What is the highest annual spend of any customer?',\n", + " 'db': 'sqlite',\n", + " 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'},\n", + " {'step': 16,\n", + " 'tool': 'sql',\n", + " 'db_response': [('Melinda Lyons',), ('Melinda Lyons',)],\n", + " 'sql_query': 'SELECT customer_name FROM customer_table WHERE account_number = 1234953',\n", + " 'query': 'Which customer has account number 1234953',\n", + " 'db': 'sqlite',\n", + " 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'},\n", + " {'step': 21,\n", + " 'tool': 'sql',\n", + " 'db_response': [('Michael Rogers',)],\n", + " 'sql_query': 'SELECT customer_name FROM customer_table ORDER BY annual_spend LIMIT 1',\n", + " 'query': 'Which customer has the lowest annual spend?',\n", + " 'db': 'sqlite',\n", + " 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'},\n", + " {'step': 26,\n", + " 'tool': 'sql',\n", + " 'db_response': [('no',), ('no',)],\n", + " 'sql_query': \"SELECT vip_customer FROM customer_table WHERE customer_name = 'Susan Soinsin'\",\n", + " 'query': 'Is Susan Soinsin a vip customer?',\n", + " 'db': 'sqlite',\n", + " 'work_item': 'CREATE TABLE customer_table (customer_name text, account_number integer, customer_level text, vip_customer text, annual_spend integer, user_name text )'}]" + ] + }, + "execution_count": 38, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "query_list = [\n", + " \"Which customers have vip customer status of yes?\",\n", + " \"What is the highest annual spend of any customer?\",\n", + " \"Which customer has account number 1234953\",\n", + " \"Which customer has the lowest annual spend?\",\n", + " \"Is Susan Soinsin a vip customer?\"\n", + "]\n", + "\n", + "# Execute the queries\n", + "agent_natural_language_sql_query(query_list, db=db, table_name=table_name)\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/NoteBook_Examples/document_summarizer_notebook.ipynb b/tutorials/notebooks/NoteBook_Examples/document_summarizer_notebook.ipynb new file mode 100644 index 0000000..562e731 --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/document_summarizer_notebook.ipynb @@ -0,0 +1,339 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Document Summaries with Small Language Models (SLIMs Cookbook)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The need for document summaries\n", + "\n", + "Often, we face an issue with smaller models, where the context window of the model is relatively small. If we want to pass in entire documents to the model that exceed the context window, we need to find a way to summarize the contents of the documents. Let's look at an example of how to do this with LLMWare." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### For Google Colab users" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are using Colab for free, we highly recommend you activate the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as apposed to the ~13GB Colab gives automatically.\n", + "\n", + "To activate T4:\n", + "1. click on the \"Runtime\" tab\n", + "2. click on \"Change runtime type\"\n", + "3. select T4 GPU under Hardware Accelerator\n", + "\n", + "NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installing and importing dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulting to user installation because normal site-packages is not writeable\n", + "Requirement already satisfied: llmware in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (0.2.14)\n", + "Requirement already satisfied: boto3==1.24.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.24.53)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.19.4)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.26.0)\n", + "Requirement already satisfied: openai>=1.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.13.3)\n", + "Requirement already satisfied: pymongo>=4.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.7.2)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.15.2)\n", + "Requirement already satisfied: torch>=1.13.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (2.1.0+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.38.2)\n", + "Requirement already satisfied: Wikipedia-API==0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.6.0)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: einops==0.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.7.0)\n", + "Requirement already satisfied: librosa>=0.10.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: botocore<1.28.0,>=1.27.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (1.27.96)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (0.6.2)\n", + "Requirement already satisfied: typing-extensions>=4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (4.8.0)\n", + "Requirement already satisfied: tzdata in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (2023.3)\n", + "Requirement already satisfied: requests in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from Wikipedia-API==0.6.0->llmware) (2.31.0)\n", + "Requirement already satisfied: filelock in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (3.13.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (2023.10.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (4.66.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (23.1)\n", + "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.11.3)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.1)\n", + "Requirement already satisfied: joblib>=0.14 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (5.1.1)\n", + "Requirement already satisfied: numba>=0.51.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.59.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (4.0.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (0.23.3)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (2.6.4)\n", + "Requirement already satisfied: sniffio in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (1.3.0)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: sympy in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (3.2.1)\n", + "Requirement already satisfied: jinja2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (3.1.2)\n", + "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from transformers>=4.36.0->llmware) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from transformers>=4.36.0->llmware) (0.4.2)\n", + "Requirement already satisfied: idna>=2.8 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (2.10)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.26.18)\n", + "Requirement already satisfied: certifi in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2023.7.22)\n", + "Requirement already satisfied: httpcore<0.17.0,>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.16.3)\n", + "Requirement already satisfied: rfc3986<2,>=1.3 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from rfc3986[idna2008]<2,>=1.3->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (1.5.0)\n", + "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.42.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (3.10.0)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.16.3 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.16.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.2.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.2.0)\n", + "Requirement already satisfied: cffi>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.15.1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from jinja2->torch>=1.13.1->llmware) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.21)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpcore<0.17.0,>=0.15.0->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.14.0)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from llmware.prompts import Prompt\n", + "from llmware.setup import Setup" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Worker function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The process for summarizing the document is as follows:\n", + "- The document gets split up into batches (each batch can be a few pages long)\n", + "- The SLIM Summary tool summarizes each batch into a list of points\n", + "- The points from each batch are aggregated at the end and printed, and the output is also available as a Python list\n", + "\n", + "The LLMWare library provides several sample files for you to run this example with, but if you wanted to use your own documents, then you can pass in the following optional parameters for the summarization:\n", + "- Topic: all the points in the summary will be about this topic\n", + "- Query: it is used to filter which parts of the document are considered for the summarization (only text chunks matching the query will be considered)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def test_summarize_document(example=\"jd salinger\"):\n", + "\n", + " # pull a sample document (or substitute a file_path and file_name of your own)\n", + " sample_files_path = Setup().load_sample_files(over_write=False)\n", + "\n", + " topic = None\n", + " query = None\n", + " fp = None\n", + " fn = None\n", + "\n", + " if example not in [\"jd salinger\", \"employment terms\", \"just the comp\", \"un resolutions\"]:\n", + " print (\"not found example\")\n", + " return []\n", + "\n", + " if example == \"jd salinger\":\n", + " fp = os.path.join(sample_files_path, \"SmallLibrary\")\n", + " fn = \"Jd-Salinger-Biography.docx\"\n", + " topic = \"jd salinger\"\n", + " query = None\n", + "\n", + " if example == \"employment terms\":\n", + " fp = os.path.join(sample_files_path, \"Agreements\")\n", + " fn = \"Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf\"\n", + " topic = \"executive compensation terms\"\n", + " query = None\n", + "\n", + " if example == \"just the comp\":\n", + " fp = os.path.join(sample_files_path, \"Agreements\")\n", + " fn = \"Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf\"\n", + " topic = \"executive compensation terms\"\n", + " query = \"base salary\"\n", + "\n", + " if example == \"un resolutions\":\n", + " fp = os.path.join(sample_files_path, \"SmallLibrary\")\n", + " fn = \"N2126108.pdf\"\n", + " # fn = \"N2137825.pdf\"\n", + " topic = \"key points\"\n", + " query = None\n", + "\n", + " # optional parameters: 'query' - will select among blocks with the query term\n", + " # 'topic' - will pass a topic/issue as the parameter to the model to 'focus' the summary\n", + " # 'max_batch_cap' - caps the number of batches sent to the model\n", + " # 'text_only' - returns just the summary text aggregated\n", + "\n", + " kp = Prompt().summarize_document_fc(fp, fn, topic=topic, query=query, text_only=True, max_batch_cap=15)\n", + "\n", + " print(f\"\\nDocument summary completed - {len(kp)} Points\")\n", + " for i, points in enumerate(kp):\n", + " print(i, points)\n", + "\n", + " return 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Main function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we have our function call to the worker function above." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Example: Summarize Documents\n", + "\n", + "update: Prompt - summarize_document_fc - document - Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf\n", + "update: Prompt - summarize_document_fc - number of source batches - 14\n", + "update: iterating through source batches - 0 - ['Accrued Compensation - all compensation reimbursements and other amounts earned by payable to or accrued and vested for Executive through and including Executives Date of Termination including but not limited to (i) Base Salary; (ii) Executives Incentive Bonus for the fiscal year that ended immediately prior to Executives Date of Termination to the extent such Incentive Bonus was accrued and earned by but not yet paid to Executive as of Executives Date of Termination; (iii) pay for accrued but unused vacation; and (iv) reimbursable business expenses incurred by Executive on behalf of Employer']\n", + "update: iterating through source batches - 1 - ['Not Found']\n", + "update: iterating through source batches - 2 - ['Not to exceed 5% of base salary', 'Material Competitor', 'Sale of the Company', 'Material breach of the terms of the agreement']\n", + "update: iterating through source batches - 3 - ['Base Salary: $500,000 annual rate', 'Target Bonus: 80% of base salary (\"Target Bonus\")', 'Incentive Bonus: 125% of base salary (\"Incentive Bonus\")', 'Sale Bonus: 2.5% of the equity value of the company, up to $1,000,000', 'Transition Bonus: $350,000, payable on October 1, 2013 and October 1, 2014']\n", + "update: iterating through source batches - 4 - ['25 vacation days', '7 fixed holidays', '7 floating holidays', 'reimbursement of reasonable expenses', 'Initial stock options grant', 'Severance of up to 12 months salary and severance bonus if employment is terminated without Cause or resignation for Good Reason']\n", + "update: iterating through source batches - 5 - ['Severance Executive will receive severance in monthly installments in accordance with Employers regular payroll practices during the Severance Period', 'Sale Bonus If the Executives employment is terminated by the Company without Cause or due to Executives resignation for Good Reason prior to the payment of the First Transition Bonus then the Company will pay the Executive a lump-sum cash amount equal to the First Transition Bonus on the First Payment Date', 'Health Care Continuation If Executive is eligible for and elects to receive continuation group health coverage mandated by Section 4980B of the Internal Revenue Code or similar state laws (', ') during the Severance Period then the Company will reimburse Executive for the amount of the COBRA premiums and the Company will reimburse Executive for the amount of the COBRA premiums']\n", + "update: iterating through source batches - 6 - ['Not Found']\n", + "update: iterating through source batches - 7 - ['Not Found']\n", + "update: iterating through source batches - 8 - ['Not Found']\n", + "update: iterating through source batches - 9 - ['Not Found']\n", + "update: iterating through source batches - 10 - ['Not Found']\n", + "update: iterating through source batches - 11 - ['Not Found']\n", + "update: iterating through source batches - 12 - ['Notwithstanding any provision of the agreement to the contrary, if the Executive is a \"specified employee\" at the time of separation from service, then the payments or benefits otherwise payable pursuant to the agreement will be postponed to prevent any accelerated or additional tax under Section 409A of the Code', 'If any payments or benefits are postponed due to Section 409A of the Code, then the Executive will be entitled to be paid interest on such amount at an annual rate equal to the prime rate plus 2%', 'If the Executive dies during the postponement period, then the amounts withheld on account of Section 409A of the Code will be paid to the estate of the Executive within sixty days after the date of death']\n", + "update: iterating through source batches - 13 - ['Not Found']\n", + "\n", + "Document summary completed - 23 Points\n", + "0 Accrued Compensation - all compensation reimbursements and other amounts earned by payable to or accrued and vested for Executive through and including Executives Date of Termination including but not limited to (i) Base Salary; (ii) Executives Incentive Bonus for the fiscal year that ended immediately prior to Executives Date of Termination to the extent such Incentive Bonus was accrued and earned by but not yet paid to Executive as of Executives Date of Termination; (iii) pay for accrued but unused vacation; and (iv) reimbursable business expenses incurred by Executive on behalf of Employer\n", + "1 Not to exceed 5% of base salary\n", + "2 Material Competitor\n", + "3 Sale of the Company\n", + "4 Material breach of the terms of the agreement\n", + "5 Base Salary: $500,000 annual rate\n", + "6 Target Bonus: 80% of base salary (\"Target Bonus\")\n", + "7 Incentive Bonus: 125% of base salary (\"Incentive Bonus\")\n", + "8 Sale Bonus: 2.5% of the equity value of the company, up to $1,000,000\n", + "9 Transition Bonus: $350,000, payable on October 1, 2013 and October 1, 2014\n", + "10 25 vacation days\n", + "11 7 fixed holidays\n", + "12 7 floating holidays\n", + "13 reimbursement of reasonable expenses\n", + "14 Initial stock options grant\n", + "15 Severance of up to 12 months salary and severance bonus if employment is terminated without Cause or resignation for Good Reason\n", + "16 Severance Executive will receive severance in monthly installments in accordance with Employers regular payroll practices during the Severance Period\n", + "17 Sale Bonus If the Executives employment is terminated by the Company without Cause or due to Executives resignation for Good Reason prior to the payment of the First Transition Bonus then the Company will pay the Executive a lump-sum cash amount equal to the First Transition Bonus on the First Payment Date\n", + "18 Health Care Continuation If Executive is eligible for and elects to receive continuation group health coverage mandated by Section 4980B of the Internal Revenue Code or similar state laws (\n", + "19 ) during the Severance Period then the Company will reimburse Executive for the amount of the COBRA premiums and the Company will reimburse Executive for the amount of the COBRA premiums\n", + "20 Notwithstanding any provision of the agreement to the contrary, if the Executive is a \"specified employee\" at the time of separation from service, then the payments or benefits otherwise payable pursuant to the agreement will be postponed to prevent any accelerated or additional tax under Section 409A of the Code\n", + "21 If any payments or benefits are postponed due to Section 409A of the Code, then the Executive will be entitled to be paid interest on such amount at an annual rate equal to the prime rate plus 2%\n", + "22 If the Executive dies during the postponement period, then the amounts withheld on account of Section 409A of the Code will be paid to the estate of the Executive within sixty days after the date of death\n" + ] + } + ], + "source": [ + "if __name__ == \"__main__\":\n", + "\n", + " print(f\"\\nExample: Summarize Documents\\n\")\n", + "\n", + " # 4 examples - [\"jd salinger\", \"employment terms\", \"just the comp\", \"un resolutions\"]\n", + " # -- \"jd salinger\" - summarizes key points about jd salinger from short biography document\n", + " # -- \"employment terms\" - summarizes the executive compensation terms across 15 page document\n", + " # -- \"just the comp\" - queries to find subset of document and then summarizes the key terms\n", + " # -- \"un resolutions\" - summarizes the un resolutions document\n", + "\n", + " summary_direct = test_summarize_document(example=\"employment terms\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We're given an output of 23 points summarizing the `employment terms` sample document!\n", + "\n", + "Note that several batches gave us an output of `Not Found`, indicating that there were no relevant points related to the topic of `executive compensation terms`. This is the topic that was passed in to the `summarize_document_fc()` function call in our worker function. The model gave us the output of `Not Found` rather than unrelated points with no meaning to our topic!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/notebooks/NoteBook_Examples/msa_processing_notebook.ipynb b/tutorials/notebooks/NoteBook_Examples/msa_processing_notebook.ipynb new file mode 100644 index 0000000..44bbba5 --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/msa_processing_notebook.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Contract Analysis Using 6B Model on Laptop with LLMWare" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Imagine you were given a large contract and asked a really specific question about it: \"What is the notice for termination for convenience?\" It would be an ordeal to locate the answer for this in the contract.\n", + "\n", + "But what if we could use AI 🤖 to analyze the contract and answer this for us?\n", + "\n", + "What we want here is to perform something known as retrieval-augmented generation (RAG). This is the process by which we give a language model some external sources (such as a contract). The external sources are intended to enhance the model's context, giving it a more comprehensive understanding of a topic. The model should then give us more accurate responses to the questions we ask it on the topic.\n", + "\n", + "Now, a general purpose model like Chat-GPT might be able to answer questions about contracts with RAG, but we can do something better. We can use LLMWare's dragon-yi-6b-gguf model, which is RAG-finetuned for fact-based question-answering on complex business and legal documents." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### For Google Colab Users" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are using Colab for free, we highly recommend you activate the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as apposed to the ~13GB Colab gives automatically.\n", + "\n", + "To activate T4:\n", + "1. click on the \"Runtime\" tab\n", + "2. click on \"Change runtime type\"\n", + "3. select T4 GPU under Hardware Accelerator\n", + "\n", + "NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installing and importing dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulting to user installation because normal site-packages is not writeable\n", + "Requirement already satisfied: llmware in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (0.2.14)\n", + "Requirement already satisfied: boto3==1.24.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.24.53)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.19.4)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.26.0)\n", + "Requirement already satisfied: openai>=1.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.13.3)\n", + "Requirement already satisfied: pymongo>=4.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.7.2)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.15.2)\n", + "Requirement already satisfied: torch>=1.13.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (2.1.0+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.38.2)\n", + "Requirement already satisfied: Wikipedia-API==0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.6.0)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: einops==0.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.7.0)\n", + "Requirement already satisfied: librosa>=0.10.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: botocore<1.28.0,>=1.27.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (1.27.96)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (0.6.2)\n", + "Requirement already satisfied: typing-extensions>=4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (4.8.0)\n", + "Requirement already satisfied: tzdata in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (2023.3)\n", + "Requirement already satisfied: requests in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from Wikipedia-API==0.6.0->llmware) (2.31.0)\n", + "Requirement already satisfied: filelock in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (3.13.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (2023.10.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (4.66.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (23.1)\n", + "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.11.3)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.1)\n", + "Requirement already satisfied: joblib>=0.14 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (5.1.1)\n", + "Requirement already satisfied: numba>=0.51.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.59.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (4.0.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (0.23.3)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (2.6.4)\n", + "Requirement already satisfied: sniffio in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (1.3.0)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: sympy in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (3.2.1)\n", + "Requirement already satisfied: jinja2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (3.1.2)\n", + "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from transformers>=4.36.0->llmware) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from transformers>=4.36.0->llmware) (0.4.2)\n", + "Requirement already satisfied: idna>=2.8 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (2.10)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.26.18)\n", + "Requirement already satisfied: certifi in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2023.7.22)\n", + "Requirement already satisfied: httpcore<0.17.0,>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.16.3)\n", + "Requirement already satisfied: rfc3986<2,>=1.3 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from rfc3986[idna2008]<2,>=1.3->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (1.5.0)\n", + "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.42.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (3.10.0)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.16.3 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.16.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.2.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.2.0)\n", + "Requirement already satisfied: cffi>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.15.1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from jinja2->torch>=1.13.1->llmware) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.21)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpcore<0.17.0,>=0.15.0->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.14.0)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from llmware.setup import Setup\n", + "from llmware.library import Library\n", + "from llmware.prompts import Prompt, HumanInTheLoop\n", + "from llmware.retrieval import Query\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### MongoDB" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This example requires you to have an instance of MongoDB running to store the processed contracts. The easiest way to do this is to use Docker Compose with our provided file `llmware/docker-compose_mongo_milvus.yaml`. More information on how to do this is available in our [documentation](https://github.com/llmware-ai/llmware/blob/main/docs/getting_started/working_with_docker.md)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Worker function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this function, we will do the following:\n", + "1. Loading in contracts using the `Setup` class\n", + "2. Adding them to a library using the `Library` class\n", + "3. Locate MSA files using a text query on the first page of each document using the `Query` class\n", + "4. Perform a text query to find all text containing \"termination\"\n", + "5. Pass these text chunks as a source to our _dragon-yi-6b-gguf_ model with the `Prompt` class\n", + "6. Ask the model the question \"What is the notice for termination for convenience?\"\n", + "7. Output the model's response\n", + "8. Save the model's state as a CSV with the `HumanInTheLoop` class." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def msa_processing():\n", + "\n", + " local_path = Setup().load_sample_files()\n", + " agreements_path = os.path.join(local_path, \"AgreementsLarge\")\n", + "\n", + " # create a library with all of the Agreements (~80 contracts)\n", + " msa_lib = Library().create_new_library(\"msa_lib503_635\")\n", + " msa_lib.add_files(agreements_path)\n", + "\n", + " # find the \"master service agreements\" (MSA)\n", + " q = Query(msa_lib)\n", + " query = \"master services agreement\"\n", + " results = q.text_search_by_page(query, page_num=1, results_only=False)\n", + "\n", + " # results_only = False will return a dictionary with 4 keys: {\"query\", \"results\", \"doc_ID\", \"file_source\"}\n", + " msa_docs = results[\"file_source\"]\n", + "\n", + " # load prompt/llm locally\n", + " model_name = \"llmware/dragon-yi-6b-gguf\"\n", + " prompter = Prompt().load_model(model_name)\n", + "\n", + " # analyze each MSA - \"query\" & \"llm prompt\"\n", + " for i, docs in enumerate(msa_docs):\n", + "\n", + " print(\"\\n\")\n", + " print (i+1, \"Reviewing MSA - \", docs)\n", + "\n", + " # look for the termination provisions in each document\n", + " doc_filter = {\"file_source\": [docs]}\n", + " termination_provisions = q.text_query_with_document_filter(\"termination\", doc_filter)\n", + "\n", + " # package the provisions as a source to a prompt\n", + " sources = prompter.add_source_query_results(termination_provisions)\n", + "\n", + " print(\"update: sources - \", sources)\n", + "\n", + " # call the LLM and ask our question\n", + " response = prompter.prompt_with_source(\"What is the notice for termination for convenience?\")\n", + "\n", + " # post processing fact checking\n", + " stats = prompter.evidence_comparison_stats(response)\n", + " ev_source = prompter.evidence_check_sources(response)\n", + "\n", + " for i, resp in enumerate(response):\n", + " print(\"update: llm response - \", resp)\n", + " print(\"update: compare with evidence- \", stats[i][\"comparison_stats\"])\n", + " print(\"update: sources - \", ev_source[i][\"source_review\"])\n", + "\n", + " prompter.clear_source_materials()\n", + "\n", + " # Save jsonl report with full transaction history to /prompt_history folder\n", + " print(\"\\nupdate: prompt state saved at: \", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id))\n", + "\n", + " prompter.save_state()\n", + "\n", + " # Generate CSV report for easy Human review in Excel\n", + " csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv()\n", + "\n", + " print(\"\\nupdate: csv output for human review - \", csv_output)\n", + "\n", + " return 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Main block" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we call our worker function." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "1 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Delta.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed delivered \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed delivered \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1213, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1213, 'evidence_stop_char': 1623, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 100.29986071586609}, 'time_stamp': '2024-06-21_162157', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "2 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Chi.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2011, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2814, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3618, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3618, 'evidence_stop_char': 4020, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 402, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 402, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1206, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1607, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 106.03363370895386}, 'time_stamp': '2024-06-21_162343', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2011, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2814, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3618, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3618, 'evidence_stop_char': 4020, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Chi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "3 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Mu.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'any provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3617, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3617, 'evidence_stop_char': 4019, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 802, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 802, 'evidence_stop_char': 1205, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1205, 'evidence_stop_char': 1608, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 6, 'total': 1122, 'metric': 'tokens', 'processing_time': 120.17748379707336}, 'time_stamp': '2024-06-21_162544', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3617, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3617, 'evidence_stop_char': 4019, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf': [5, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Mu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "4 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Psi.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\"], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2009, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2009, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 1}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 766, 'output': 6, 'total': 772, 'metric': 'tokens', 'processing_time': 86.09340858459473}, 'time_stamp': '2024-06-21_162710', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2009, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2009, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf': [5, 6, 4]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Psi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "5 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Rho.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\"], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 1}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 766, 'output': 6, 'total': 772, 'metric': 'tokens', 'processing_time': 90.99819278717041}, 'time_stamp': '2024-06-21_162841', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1608, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1608, 'evidence_stop_char': 2011, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2814, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf': [5, 6, 4]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Rho.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "6 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Beta.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1204, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1204, 'evidence_stop_char': 1605, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 124.40906977653503}, 'time_stamp': '2024-06-21_163046', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "7 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Eta.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1205, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1205, 'evidence_stop_char': 1607, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 122.25001740455627}, 'time_stamp': '2024-06-21_163249', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Eta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "8 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Alpha.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\nbelow. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 404, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 404, 'evidence_stop_char': 807, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 807, 'evidence_stop_char': 1212, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1212, 'evidence_stop_char': 1615, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 106.45800447463989}, 'time_stamp': '2024-06-21_163435', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Alpha.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "9 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Phi.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n\", ' below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices any provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3618, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3618, 'evidence_stop_char': 4019, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 400, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 400, 'evidence_stop_char': 802, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 802, 'evidence_stop_char': 1205, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1205, 'evidence_stop_char': 1608, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 6, 'total': 1122, 'metric': 'tokens', 'processing_time': 108.62869024276733}, 'time_stamp': '2024-06-21_163624', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3618, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3618, 'evidence_stop_char': 4019, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf': [5, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Phi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "10 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Nu.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n\", 'any provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties any provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4020, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1206, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1206, 'evidence_stop_char': 1609, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 6, 'total': 1122, 'metric': 'tokens', 'processing_time': 123.69110679626465}, 'time_stamp': '2024-06-21_163828', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4020, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf': [5, 6]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Nu.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "11 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Gamma.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1210, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1210, 'evidence_stop_char': 1617, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 102.44451832771301}, 'time_stamp': '2024-06-21_164011', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Gamma.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "12 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Pi.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) yea r from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\"], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1609, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1609, 'evidence_stop_char': 2011, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 1}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) yea r from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 768, 'output': 6, 'total': 774, 'metric': 'tokens', 'processing_time': 67.6502456665039}, 'time_stamp': '2024-06-21_164119', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1609, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1609, 'evidence_stop_char': 2011, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2011, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf': [5, 6, 4]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Pi.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "13 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Zeta.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\"], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1609, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1609, 'evidence_stop_char': 2010, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 1}\n", + "update: llm response - {'llm_response': '30 days written notice.', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n below. Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 766, 'output': 6, 'total': 772, 'metric': 'tokens', 'processing_time': 64.77527284622192}, 'time_stamp': '2024-06-21_164224', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 805, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 805, 'evidence_stop_char': 1207, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1207, 'evidence_stop_char': 1609, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1609, 'evidence_stop_char': 2010, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2010, 'evidence_stop_char': 2412, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2412, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf': [5, 6, 4]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice.', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Zeta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "14 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Delta.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed delivered \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed delivered \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1213, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1213, 'evidence_stop_char': 1623, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 111.97353863716125}, 'time_stamp': '2024-06-21_164416', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Delta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "15 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Kappa.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1205, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1205, 'evidence_stop_char': 1607, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 118.6111249923706}, 'time_stamp': '2024-06-21_164615', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Kappa.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "16 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Omega.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) yea r from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) yea r from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 400, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 400, 'evidence_stop_char': 801, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 801, 'evidence_stop_char': 1204, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1204, 'evidence_stop_char': 1607, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 116.53554511070251}, 'time_stamp': '2024-06-21_164811', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omega.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "17 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Epsilon.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1205, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1205, 'evidence_stop_char': 1607, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 117.87502908706665}, 'time_stamp': '2024-06-21_165009', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "18 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Epsilon.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1205, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1205, 'evidence_stop_char': 1607, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 120.23511362075806}, 'time_stamp': '2024-06-21_165210', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Epsilon.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "19 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Omicron.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n Each of the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1204, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1204, 'evidence_stop_char': 1605, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 111.38872861862183}, 'time_stamp': '2024-06-21_165401', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2815, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2815, 'evidence_stop_char': 3216, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3216, 'evidence_stop_char': 3619, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3619, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Omicron.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "\n", + "20 Reviewing MSA - C:\\Users\\prash\\llmware_data\\tmp\\parser_tmp\\process_pdf_files\\Testco MSA Beta.pdf\n", + "update: sources - {'text_batch': [\"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", ' of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products of the Customer, which includes trade secrets. Accordingly, in order to prevent TestCo and its employees from intentionally or unintentionally misappropriating any residual Confidential Information, TestCo agr ees that for the period of one (1) year from the termination of this Agreement, its employees who provide services hereunder will not work on any Competitive Products \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\nof the contracting parties agrees to notify the other in writing of address or contact changes. All notices, authorizations, and requests given or made in connection with this agreement, including notice of termination of this agreement, must be sent by post, express courier, facsimile, or email to the addresses and numbers indicated in this section. Notices will be deemed \\n'], 'metadata_batch': [[{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 401, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 401, 'evidence_stop_char': 803, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 4, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 803, 'evidence_stop_char': 1204, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1204, 'evidence_stop_char': 1605, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 6, 'doc_id': 1, 'block_id': 1}]], 'batches_count': 2}\n", + "update: llm response - {'llm_response': '30 days written notice', 'prompt': 'What is the notice for termination for convenience?', 'evidence': \"must be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\nmust be brought within two years from the date that the cause of action arose. Term and Termination of Agreement; Assignment. This Agreement shall remain in effect until terminated. Either party may terminate this agreement, any Statement of Work or Services Description for convenience by giving the other party 30 days written notice. Either party may terminate this Agreement or \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n party may be deemed notice of termination of this Agreement, effective on the date of assignment, by the other party. Survival upon Termination or End of Term. The provisions regarding warranty, limitation of liability, confidentiality, fees and expenses, obligations on termination or expiration, ownership and license, and miscellaneous of this Agreement, and \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n any work order or services description if the other party is in material breach or default of any obligation that is not cured within 15 days' notice of such breach. The TestCo agrees to pay all fees for services performed and expenses incurred prior to the termination of this Agreement. Termination of this Agreement will terminate all outstanding Statement \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\n(24) months following termination of this Agreement, directly or indirectly, call on or attempt to call on, hire, solicit, or induce any change in or cessation of, the business relationship, of any customers, clients, contractors, vendors, contract manufacturers, suppliers, investors or employees of other on whom the party called on or became acquainted \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\nany provisions specified as surviving in a Statement of Work or Services Description, survive any termination or expiration of this agreement, any Statement of Work or Services Description. Severability. If a court holds any provision of this Agreement to be illegal, invalid or unenforceable, the remaining provisions will remain in full force and effect and the parties \\n\", 'instruction': 'default_with_context', 'model': 'dragon-yi-6b-gguf', 'usage': {'input': 1116, 'output': 5, 'total': 1121, 'metric': 'tokens', 'processing_time': 112.91939091682434}, 'time_stamp': '2024-06-21_165554', 'calling_app_ID': '', 'rating': '', 'account_name': 'llmware', 'prompt_id': '1b84f914-d851-4a83-8b13-47cd464d92db', 'batch_id': 0, 'evidence_metadata': [{'batch_source_id': 0, 'evidence_start_char': 0, 'evidence_stop_char': 403, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 1, 'evidence_start_char': 403, 'evidence_stop_char': 806, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 2, 'evidence_start_char': 806, 'evidence_stop_char': 1208, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 3, 'evidence_start_char': 1208, 'evidence_stop_char': 1610, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 4, 'evidence_start_char': 1610, 'evidence_stop_char': 2012, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 5, 'evidence_start_char': 2012, 'evidence_stop_char': 2414, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 6, 'evidence_start_char': 2414, 'evidence_stop_char': 2816, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 7, 'evidence_start_char': 2816, 'evidence_stop_char': 3218, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 8, 'evidence_start_char': 3218, 'evidence_stop_char': 3620, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}, {'batch_source_id': 9, 'evidence_start_char': 3620, 'evidence_stop_char': 4022, 'source_name': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}], 'biblio': {'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf': [5]}, 'event_type': 'inference', 'human_feedback': '', 'human_assessed_accuracy': '', 'comparison_stats': {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}, 'source_review': [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]}\n", + "update: compare with evidence- {'percent_display': '100.0%', 'confirmed_words': ['30', 'days', 'written', 'notice'], 'unconfirmed_words': [], 'verified_token_match_ratio': 1.0, 'key_point_list': [{'key_point': '30 days written notice', 'entry': 0, 'verified_match': 1.0}]}\n", + "update: sources - [{'text': 'Services Description for convenience by giving the other party 30 days written notice Either party may terminate this Agreement or ', 'match_score': 1.0, 'source': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\tmp\\\\parser_tmp\\\\process_pdf_files\\\\Testco MSA Beta.pdf', 'page_num': 5, 'doc_id': 1, 'block_id': 1}]\n", + "\n", + "update: prompt state saved at: C:\\Users\\prash\\llmware_data\\prompt_history\\1b84f914-d851-4a83-8b13-47cd464d92db\n", + "\n", + "update: csv output for human review - {'report_name': 'interaction_report_2024-06-21_165554.csv', 'report_fp': 'C:\\\\Users\\\\prash\\\\llmware_data\\\\prompt_history\\\\interaction_report_2024-06-21_165554.csv', 'results': 20}\n" + ] + } + ], + "source": [ + "if __name__ == \"__main__\":\n", + "\n", + " m = msa_processing()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our output contains a dictionary for each LLM response with several keys, notably:\n", + "- `llm_response`: giving us the answer to our question\n", + "- `evidence`: giving us the text where the model found the answer to the question" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/notebooks/NoteBook_Examples/ner_retrieval.ipynb b/tutorials/notebooks/NoteBook_Examples/ner_retrieval.ipynb new file mode 100644 index 0000000..0f3e1d4 --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/ner_retrieval.ipynb @@ -0,0 +1,2475 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "L3jJhdKW5Oeo" + }, + "source": [ + "# **If you are using Colab for free, we highly recommend you active the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed to the 13GB Colab gives automatically.**\n", + "# **To active T4:**\n", + "# **1. click on the \"Runtime\" tab**\n", + "# **2. click on \"Change runtime type\"**\n", + "# **3. select T4 GPU under Hardware Accelerator**\n", + "# **NOTE: there is a weekly usage limit on using T4 for free**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lxFs5ZL5O2jZ", + "outputId": "ee864ee3-5de8-4bf7-8327-935258471852" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.3.0-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.120-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.3/139.3 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (669 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m669.1/669.1 kB\u001b[0m \u001b[31m17.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "Collecting botocore<1.35.0,>=1.34.120 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.120-py3-none-any.whl (12.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m49.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.1-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.2/82.2 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.120 botocore-1.34.120 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.0 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.3 s3transfer-0.10.1\n", + "Collecting wikipedia-api\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from wikipedia-api) (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (2024.6.2)\n", + "Installing collected packages: wikipedia-api\n", + "Successfully installed wikipedia-api-0.6.0\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install wikipedia-api" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-kgUJQnhO9wY" + }, + "outputs": [], + "source": [ + "from llmware.agents import LLMfx\n", + "from llmware.parsers import WikiParser" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "PA4O0nmHPGm0" + }, + "outputs": [], + "source": [ + "text = (\"The new Miko Marks album is one of the best I have ever heard in a number of years. \"\n", + " \"She is definitely an artist worth exploring further.\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 611, + "referenced_widgets": [ + "b5ba93edf3694b91b6c022357bc63c5f", + "3ad08a5184024a61867f53cd60967b8a", + "4610da9b45a14e37a5d22437b6c80af1", + "da05759d64dd4fde96f2571ab57c10d7", + "a3b4c3736ec44e05b0408a77328afc46", + "2f422e29743147d589f78e88a5023584", + "04a584f3e21e437392ea4e1f2e454256", + "7e3bc21eddef4a3e9711034215666fc1", + "e78047ec9ef14fe0b33d559b7d1a98c1", + "7a3edae3026743adaa4dfc750652e612", + "a285f26081e54e5485d6ba816dccab04", + "3d75de815f1d4b0ba7f45ba62bd0fee0", + "98fd25c941ed4cc48b64265f9b02e21d", + "0b4e656fbf75456087a719a1f5cfe7f9", + "912f9795761d477b80183838f87798c3", + "d62e6155e43143d384c97fa637800900", + "27e955a826c44c4f888f94fd8d214342", + "d84ecb5ba26d42fca5c0a71f3557ec77", + "11f553b8d131488e918823421766447f", + "af317516d22447efb596af73d29ed7f3", + "1b767e77b41b4b2a95cea950a91140e9", + "1aae4ab030be4d19bab991a878e260ae", + "38399b2088e54e29b6fe46d61cfe375e", + "2dd4eae5448c4c0f8c4210941b32fe02", + "c51a57f4bf0645efbfb294f9c2560a58", + "f7af4381065f44f08497189171e5fa32", + "93ea85bd3d5f4a559c10411360aeb9f0", + "4e1dcfb6582741febec7ad363e39cec1", + "2cf7635b38d5497eb537461c45fc172e", + "019e216bc5474a5ba5e4bc7028b83f8f", + "38ee479136694284ba995e82fe9c2c97", + "0227bbda2303417282d010d7df429eb4", + "2775aacd793e4d26bb4ebb4570aa79cd", + "a7a23c114b8e4275bfd18872e8347cc7", + "7ce0ea72f4614748aa436eb323388ca7", + "44917c6036de44fcb5ca9bc468964caf", + "de6808042e2c49f991a66ee27bb66b05", + "c36923294558418cab03cb76147602fd", + "206fd63bc81c40648b38e2f716275c95", + "0e8834fb34e34e189a80265a23b04a0a", + "722725fd61d54b70b7124d46988ff309", + "a1ebc53841d14cd1a2fb40e7b2a28f28", + "179c1a00853d45859b1200c7385deadd", + "3a0d0a8cf1b24aeb95ef6b3df3a67bad", + "cb53876e1e9941c1be09b31acc33022c", + "e559393bb20441ac99f79e7b665e6466", + "0784e58f828141d38104fb7c8b6d37fa", + "3941ca3615e349f9abc379840d7ad5d7", + "66fa1a4bfe684b3296e9733209a3e755", + "1f26676cf8a0472d82edf15e1605fd07", + "90bccb0588ac4dbfb1c6ebff10663214", + "a9075f2798d94525861de0c205b67ad0", + "edf56f8ea6044bf7a4c782e767216299", + "40ae101d8d824a4d9a45cc7eb2f4d23d", + "666ecaec0c19436eba7ffb801316707a", + "c5e77bad53e64c398568b55814c805b0", + "87f4f58671ad419c998dce7641f63ae7", + "7b3eb01c9d0e4e51b12ec860020a5c9c", + "d52782db557244b6a1a0987cbe11afbe", + "9417ecffee87490b98a0515b9c90152f", + "4de8ad290b3f4ad88ae9dd82bd2db17b", + "6f81e12919f74ecfa091f0a70920ea1b", + "5ea39357b9ca4d439469d4b0fdb7dc06", + "693871dab6e84a78a709fa5f64a62631", + "3f10128eefff4dbc91af23ca3b644fb8", + "737f4afd766d44b192db1cb921504334" + ] + }, + "id": "Ue0d-BL8PmpT", + "outputId": "3f4803eb-9371-4905-d6ac-ad69affae2c8" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Launching LLMfx process\n", + "step - \t1 - \tcreating object - ready to start processing.\n", + "step - \t2 - \tloading new processing text - 1 new entries\n", + "step - \t3 - \tloading tool - ner\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "b5ba93edf3694b91b6c022357bc63c5f", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00=0.19.4 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (1.26.4)\n", + "Requirement already satisfied: openai>=1.0.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (1.30.3)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (4.7.2)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.15.2)\n", + "Requirement already satisfied: torch>=1.13.1 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (2.2.1)\n", + "Requirement already satisfied: transformers>=4.36.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (4.39.1)\n", + "Requirement already satisfied: Wikipedia-API==0.6.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.6.0)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: einops==0.7.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.7.0)\n", + "Requirement already satisfied: librosa>=0.10.0 in /opt/homebrew/lib/python3.11/site-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: botocore<1.28.0,>=1.27.53 in /opt/homebrew/lib/python3.11/site-packages (from boto3==1.24.53->llmware) (1.27.96)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/homebrew/lib/python3.11/site-packages (from boto3==1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/homebrew/lib/python3.11/site-packages (from boto3==1.24.53->llmware) (0.6.2)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /opt/homebrew/lib/python3.11/site-packages (from psycopg==3.1.17->llmware) (4.10.0)\n", + "Requirement already satisfied: requests in /opt/homebrew/lib/python3.11/site-packages (from Wikipedia-API==0.6.0->llmware) (2.32.2)\n", + "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->llmware) (2024.3.1)\n", + "Requirement already satisfied: packaging>=20.9 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface-hub>=0.19.4->llmware) (4.66.2)\n", + "Requirement already satisfied: audioread>=2.1.9 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (1.12.0)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (1.5.0)\n", + "Requirement already satisfied: joblib>=0.14 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (5.1.1)\n", + "Requirement already satisfied: numba>=0.51.0 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (0.59.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /opt/homebrew/lib/python3.11/site-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->llmware) (4.4.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->llmware) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->llmware) (0.27.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->llmware) (2.7.1)\n", + "Requirement already satisfied: sniffio in /opt/homebrew/lib/python3.11/site-packages (from openai>=1.0.0->llmware) (1.3.1)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /opt/homebrew/lib/python3.11/site-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: sympy in /opt/homebrew/lib/python3.11/site-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in /opt/homebrew/lib/python3.11/site-packages (from torch>=1.13.1->llmware) (3.3)\n", + "Requirement already satisfied: jinja2 in /opt/homebrew/lib/python3.11/site-packages (from torch>=1.13.1->llmware) (3.1.4)\n", + "Requirement already satisfied: regex!=2019.12.17 in /opt/homebrew/lib/python3.11/site-packages (from transformers>=4.36.0->llmware) (2024.5.15)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /opt/homebrew/lib/python3.11/site-packages (from transformers>=4.36.0->llmware) (0.4.3)\n", + "Requirement already satisfied: idna>=2.8 in /opt/homebrew/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (3.7)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/homebrew/lib/python3.11/site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.9.0.post0)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/homebrew/lib/python3.11/site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.26.18)\n", + "Requirement already satisfied: certifi in /opt/homebrew/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in /opt/homebrew/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (1.0.5)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /opt/homebrew/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.14.0)\n", + "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in /opt/homebrew/lib/python3.11/site-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.42.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /opt/homebrew/lib/python3.11/site-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /opt/homebrew/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /opt/homebrew/lib/python3.11/site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.3.2)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in /opt/homebrew/lib/python3.11/site-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.4.0)\n", + "Requirement already satisfied: cffi>=1.0 in /opt/homebrew/lib/python3.11/site-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /opt/homebrew/lib/python3.11/site-packages (from jinja2->torch>=1.13.1->llmware) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /opt/homebrew/lib/python3.11/site-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in /opt/homebrew/lib/python3.11/site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /opt/homebrew/lib/python3.11/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "809f3d5a-37f9-4843-b3dd-080a10e9d8c9", + "metadata": {}, + "outputs": [], + "source": [ + "from llmware.models import ModelCatalog\n", + "from llmware.parsers import WikiParser\n", + "\n", + "\n", + "# our input - financial news article - note: the 'company founding date' is not mentioned in the text\n", + "# -- our worry is that the model will either 'make up' an answer from background knowledge (which may or may not\n", + "# be accurate), or try to incorrectly use a date in the text, e.g, February 29, 2024.\n", + "\n", + "\n", + "text =(\"BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its \"\n", + " \"third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter \"\n", + " \"of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* \"\n", + " \"at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis \"\n", + " \"NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis \"\n", + " \"Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin \"\n", + " \"increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges \"\n", + " \"Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring \"\n", + " \"charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these \"\n", + " \"charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to \"\n", + " \"drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by \"\n", + " \"the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and \"\n", + " \"work with our wholesale partners to elevate and grow the marketplace.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "b6c5f0ab-e5d7-41da-8487-15cd1647962e", + "metadata": {}, + "outputs": [], + "source": [ + "def not_found_then_triage_lookup():\n", + "\n", + " print(\"\\nNot Found Example - if info not found, then lookup in another source.\\n\")\n", + "\n", + " extract_key = \"company founding date\"\n", + " dict_key = extract_key.replace(\" \", \"_\")\n", + "\n", + " company_founding_date = \"\"\n", + "\n", + " # step 1 - run an extract function call on the text\n", + " model = ModelCatalog().load_model(\"slim-extract-tool\", temperature=0.0, sample=False)\n", + " response = model.function_call(text, function=\"extract\", params=[extract_key])\n", + " llm_response = response[\"llm_response\"]\n", + "\n", + " print(f\"update: first text reviewed for {extract_key} - llm response: \", llm_response)\n", + "\n", + " # unpack the output\n", + " if dict_key in llm_response:\n", + "\n", + " company_founding_date = llm_response[dict_key]\n", + "\n", + " if len(company_founding_date) > 0:\n", + "\n", + " # in this case, the value is a list with at least one element, so an 'answer' was found\n", + " company_founding_date = company_founding_date[0]\n", + " print(f\"update: found the {extract_key} value - \", company_founding_date)\n", + " return company_founding_date\n", + "\n", + " else:\n", + "\n", + " # step 2 - could not find the answer in the original source materials\n", + " # e.g., the len of the list associated with the key is zero, or []\n", + "\n", + " print(f\"update: did not find the target value in the text - {company_founding_date}\")\n", + " print(\"update: initiating a secondary process to try to find the information\")\n", + "\n", + " # look up the company name from the original text\n", + " response = model.function_call(text, function=\"extract\", params=[\"company name\"])\n", + "\n", + " if \"company_name\" in response[\"llm_response\"]:\n", + " company_name = response[\"llm_response\"][\"company_name\"][0]\n", + "\n", + " if company_name:\n", + " print(f\"\\nupdate: found the company name - {company_name} - now using to lookup in secondary source\")\n", + "\n", + " # use the company name to lookup materials in Wikipedia secondary source\n", + " output = WikiParser().add_wiki_topic(company_name,target_results=1)\n", + "\n", + " if output:\n", + "\n", + " supplemental_text = output[\"articles\"][0][\"summary\"]\n", + "\n", + " if len(supplemental_text) > 150:\n", + " supplemental_text_pp = supplemental_text[0:150] + \" ... \"\n", + " else:\n", + " supplemental_text_pp = supplemental_text\n", + "\n", + " print(f\"update: using lookup - {company_name} - found secondary source article \"\n", + " f\"(extract displayed) - \", supplemental_text_pp)\n", + "\n", + " # finally, try to get the company founding date from the supplemental text\n", + " new_response = model.function_call(supplemental_text,params=[\"company founding date\"])\n", + "\n", + " print(\"\\nupdate: reviewed second source article - \", new_response[\"llm_response\"])\n", + "\n", + " if \"company_founding_date\" in new_response[\"llm_response\"]:\n", + " company_founding_date = new_response[\"llm_response\"][\"company_founding_date\"]\n", + " if company_founding_date:\n", + " print(\"update: success - found the answer - \", company_founding_date)\n", + "\n", + " return company_founding_date" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "296a32b1-db9f-4e09-9c6f-ab4c40046634", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Not Found Example - if info not found, then lookup in another source.\n", + "\n", + "update: first text reviewed for company founding date - llm response: {'company_founding_date': []}\n", + "update: did not find the target value in the text - []\n", + "update: initiating a secondary process to try to find the information\n", + "\n", + "update: found the company name - NIKE, Inc. - now using to lookup in secondary source\n", + "update: using lookup - NIKE, Inc. - found secondary source article (extract displayed) - Nike, Inc. (stylized as NIKE) is an American athletic footwear and apparel corporation headquartered near Beaverton, Oregon, United States. It is the ... \n", + "\n", + "update: reviewed second source article - {'company_founding_date': ['January 25, 1964']}\n", + "update: success - found the answer - ['January 25, 1964']\n" + ] + } + ], + "source": [ + "if __name__ == \"__main__\":\n", + "\n", + " founding_date = not_found_then_triage_lookup()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ddd92bbb-1f24-4684-b0ca-61d0e0a26ff9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tutorials/notebooks/NoteBook_Examples/parsing_great_speeches_notebook.ipynb b/tutorials/notebooks/NoteBook_Examples/parsing_great_speeches_notebook.ipynb new file mode 100644 index 0000000..3675a2e --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/parsing_great_speeches_notebook.ipynb @@ -0,0 +1,229 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Voice Transcription with CPU Friendly AI Models Example (Greatest Speeches of 20th Century)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If we were given an audio file, is there any way we could identify the time stamps where specific words were said? Is there any way we could extract all the key words mentioned about a topic?\n", + "\n", + "With AI 🤖, we can do all of this and much more! The key lies in being able to parse audio into text, allowing us to then harness the natural language processing capabilities of language models to perform sophisticated analyses and inferences on our data.\n", + "\n", + "Regardless of who you are, such an approach to audio transcription and analysis will augment how you interact with and extract knowledge from audio files.\n", + "\n", + "Let's see how we can do this with llmware, using the Whisper model and the SLIM Extract Tool." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### For Google Colab Users" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are using Colab for free, we highly recommend you activate the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as apposed to the ~13GB Colab gives automatically.\n", + "\n", + "To activate T4:\n", + "1. click on the \"Runtime\" tab\n", + "2. click on \"Change runtime type\"\n", + "3. select T4 GPU under Hardware Accelerator\n", + "\n", + "NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installing and importing dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from llmware.parsers import Parser\n", + "from llmware.setup import Setup\n", + "from llmware.util import Utilities\n", + "from llmware.models import ModelCatalog" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Worker function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Our main worker function will do the following:\n", + "1. Load in sample audio files with the `Setup` class\n", + "2. Parse the audio into text by calling the WhisperCPP model via the `Parser` class\n", + "3. Perform a text search for the word \"president\" (these specific text chunks are used in the next step)\n", + "4. Use the SLIM Extract Tool to extract the president names from these chunks\n", + "5. Check if the extracted name matches someone in a specified list (`various_american_presidents`)\n", + " - If there is a match, append information about the location of the extracted name and the time stamp in the audio to a `final_list`\n", + "6. Output the content of the `final_list`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def greatest_speeches_example():\n", + "\n", + " # four sample file sets - \"famous_quotes\" | \"greatest_speeches\" | \"youtube_demos\" | \"earnings_calls\"\n", + " voice_sample_files = Setup().load_voice_sample_files(small_only=False)\n", + "\n", + " input_folder = os.path.join(voice_sample_files, \"greatest_speeches\")\n", + "\n", + " print(\"\\nStep 1 - converting, parsing and text chunking ~56 WAV files\")\n", + "\n", + " parser_output = Parser(chunk_size=400, max_chunk_size=600).parse_voice(input_folder,\n", + " write_to_db=False,\n", + " copy_to_library=False,\n", + " remove_segment_markers=True,\n", + " chunk_by_segment=True,\n", + " real_time_progress=False)\n", + "\n", + " print(\"\\nStep 2- look at the text chunks with all of the metadata\")\n", + "\n", + " for i, entries in enumerate(parser_output):\n", + " print(\"all parsed blocks: \", i, entries)\n", + "\n", + " print(\"\\nStep 3- run an inline text search for 'president'\")\n", + "\n", + " results = Utilities().fast_search_dicts(\"president\", parser_output)\n", + "\n", + " for i, res in enumerate(results):\n", + " print(\"search results: \", i, res)\n", + "\n", + " print(\"\\nStep 4- use LLM to review each search result - and if specific U.S. presidents found, then display the source\")\n", + "\n", + " extract_model = ModelCatalog().load_model(\"slim-extract-tool\", sample=False, temperature=0.0, max_output=200)\n", + "\n", + " final_list = []\n", + "\n", + " for i, res in enumerate(results):\n", + "\n", + " response = extract_model.function_call(res[\"text\"], params=[\"president name\"])\n", + "\n", + " various_american_presidents = [\"kennedy\", \"carter\", \"nixon\", \"reagan\", \"clinton\", \"obama\"]\n", + "\n", + " extracted_name = \"\"\n", + " if \"president_name\" in response[\"llm_response\"]:\n", + " if len(response[\"llm_response\"][\"president_name\"]) > 0:\n", + " extracted_name = response[\"llm_response\"][\"president_name\"][0].lower()\n", + " else:\n", + " print(\"\\nupdate: skipping result - no president name found - \", response[\"llm_response\"], res[\"text\"])\n", + "\n", + " for president in various_american_presidents:\n", + " if president in extracted_name:\n", + " print(\"\\nextracted american president text: \", i, extracted_name)\n", + " print(\"file source: \", res[\"file_source\"])\n", + " print(\"time stamp: \", res[\"coords_x\"], res[\"coords_y\"], res[\"coords_cx\"], res[\"coords_cy\"])\n", + " print(\"text: \", i, res[\"text\"])\n", + " final_list.append({\"key\": president, \"source\": res[\"file_source\"], \"time_start\": res[\"coords_x\"],\n", + " \"text\": res[\"text\"]})\n", + "\n", + " print(\"\\nStep 5 - final results\")\n", + " for i, f in enumerate(final_list):\n", + " print(\"final results: \", i, f)\n", + "\n", + " return final_list" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Main block" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we call the worker function." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "if __name__ == \"__main__\":\n", + "\n", + " greatest_speeches_example()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output, `final_list`, is a Python dictionary with information about the instances where a president name mentioned in the audio matches one of our specified presidents in the `various_american_presidents_list`. It has the following keys:\n", + "- key: the name of the president identified\n", + "- source: the audio file this was found in\n", + "- time_start: the time stamp in seconds where the president was mentioned\n", + "- text: which contains the text chunk the name was found in." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/notebooks/NoteBook_Examples/sentiment_analysis.ipynb b/tutorials/notebooks/NoteBook_Examples/sentiment_analysis.ipynb new file mode 100644 index 0000000..48ed2d4 --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/sentiment_analysis.ipynb @@ -0,0 +1,2523 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e4rVoXBpRMwI" + }, + "outputs": [], + "source": [ + "# If you are using Colab for free, we highly recommend you activate the T4 GPU\n", + "# hardware accelerator. Our models are designed to run with at least 16GB\n", + "# of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed\n", + "# to the ~13GB Colab gives automatically.\n", + "# To activate T4:\n", + "# 1. click on the \"Runtime\" tab\n", + "# 2. click on \"Change runtime type\"\n", + "# 3. select T4 GPU under \"Hardware Accelerator\"\n", + "# NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "9TasXeh9ntUx", + "outputId": "0673f3a8-126d-47e7-e2ec-1eb0943343de" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.3.0-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.124-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (669 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m669.1/669.1 kB\u001b[0m \u001b[31m21.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "Collecting botocore<1.35.0,>=1.34.124 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.124-py3-none-any.whl (12.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m58.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.1-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.2/82.2 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m28.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.124->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.124->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.124->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.124 botocore-1.34.124 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.0 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.3 s3transfer-0.10.1\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DNym3TAGnw9N" + }, + "outputs": [], + "source": [ + "from llmware.agents import LLMfx\n", + "\n", + "earnings_transcripts = [\n", + " \"This is one of the best quarters we can remember for the industrial sector with significant growth across the \"\n", + " \"board in new order volume, as well as price increases in excess of inflation. We continue to see very strong \"\n", + " \"demand, especially in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would be \"\n", + " \"accumulating more stock on any dips. \",\n", + "\n", + " \"Not the worst results, but overall we view as negative signals on the direction of the economy, and the likely \"\n", + " \"short-term trajectory for the telecom sector, and especially larger market leaders, including AT&T, Comcast, and\"\n", + " \"Deutsche Telekom.\",\n", + "\n", + " \"This quarter was a disaster for Tesla, with falling order volume, increased costs and supply, and negative \"\n", + " \"guidance for future growth forecasts in 2024 and beyond.\",\n", + "\n", + " \"On balance, this was an average result, with earnings in line with expectations and no big surprises to either \"\n", + " \"the positive or the negative.\"\n", + " ]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kwIXabPGn2cK" + }, + "outputs": [], + "source": [ + "def get_one_sentiment_classification(text):\n", + "\n", + " \"\"\"This example shows a basic use to get a sentiment classification and use the output programmatically. \"\"\"\n", + "\n", + " # simple basic use to get the sentiment on a single piece of text\n", + " agent = LLMfx(verbose=True)\n", + " agent.load_tool(\"sentiment\")\n", + " sentiment = agent.sentiment(text)\n", + "\n", + " # look at the output\n", + " print(\"sentiment: \", sentiment)\n", + " for keys, values in sentiment.items():\n", + " print(f\"{keys}-{values}\")\n", + "\n", + " # two key attributes of the sentiment output dictionary\n", + " sentiment_value = sentiment[\"llm_response\"][\"sentiment\"]\n", + " confidence_level = sentiment[\"confidence_score\"]\n", + "\n", + " # use the sentiment classification as a 'if...then' decision point in a process\n", + " if \"positive\" in sentiment_value:\n", + " print(\"sentiment is positive .... will take 'positive' analysis path ...\", sentiment_value)\n", + "\n", + " if \"positive\" in sentiment_value and confidence_level > 0.8:\n", + " print(\"sentiment is positive with high confidence ... \", sentiment_value, confidence_level)\n", + "\n", + " return sentiment" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-YqsnaLBn6SK" + }, + "outputs": [], + "source": [ + "def review_batch_earning_transcripts():\n", + "\n", + " \"\"\" This example highlights how to review multiple earnings transcripts and iterate through a batch\n", + " using the load_work mechanism. \"\"\"\n", + "\n", + " agent = LLMfx()\n", + " agent.load_tool(\"sentiment\")\n", + "\n", + " # iterating through a larger list of samples\n", + " # note: load_work method is a flexible input mechanism - pass a string, list, dictionary or combination, and\n", + " # it will 'package' as iterable units of processing work for the agent\n", + "\n", + " agent.load_work(earnings_transcripts)\n", + "\n", + " while True:\n", + " output = agent.sentiment()\n", + " # print(\"update: test - output - \", output)\n", + " if not agent.increment_work_iteration():\n", + " break\n", + "\n", + " response_output = agent.response_list\n", + "\n", + " agent.clear_work()\n", + " agent.clear_state()\n", + "\n", + " return response_output" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 765, + "referenced_widgets": [ + "5ff40e631f664081a8320d233f30c278", + "11c6a51f2e694408b1ffd01f24cd480c", + "6f325b890eaa4225b28fee5984da5af2", + "bd38cf347b554772861222fd84d1eb62", + "a3b32cf2a6964e3494b4c468c9905f91", + "8f0a9b541ba04aa6975ca3250f0cf5f0", + "9f5a3ef26bc244cc8bf6c7c82b44c52d", + "c727c4ffc69546f5920f1ac2df86450e", + "7b9f13f1baa94d2b9237e841b286f4ff", + "576c52afab1b452a99b861d997361712", + "f6d27b0e549d4ea9a687c5e29409c4f5", + "138db0b72a4048fcb99577125aa42593", + "a4436703900949eca1f49f208ba21f58", + "2582d5db708e4f83af7f6ad9ac1ddedb", + "46285793de634b409d860d33a5cc6ab2", + "49b4352eda0641d7841a6a5de7854320", + "c1cfd71b9fac4b23baec1263a6c0b2ee", + "aff951e1937042a7ae732a6ba4fb6834", + "e28c2a8c52294d88ad6d25ef66303b61", + "e1cf2fb24424404fbc2343aa2b84ed09", + "14734c0e1fbe424a963d6a52409be5f1", + "6e5bb8fc2b414127814f634ffe4d3a06", + "43bbba8de72443f3b0bd596c07116f20", + "e0302fc8a864484bba58d20931ddc3d3", + "bbb397ad4f0240b1a2e1b045935af46a", + "69be4c5935624e91a67713d370043181", + "6d549a7de0634184bc76dd9dbbbca5df", + "9973c552e9924f4a8118fd5a3d8334b5", + "39b3c03f7c4841168671573825df60d0", + "f95543b9dd9f415e86b657b199ee2773", + "202c8b18eb3841e4bc4ef74baecf6e33", + "db41b31350cf4372a0004fdd7f2750b8", + "f5b6ca2fbf6b4bcc84c73d08e6236583", + "856de742380f48c5bbbfaafa771d349a", + "326e861b92414e6b9c265888139db4de", + "96f9579890ce4146820bd808827e58f5", + "0997d67024754c73abaaf51cb3023e59", + "f4973992e12449a099d566b2caccfa1e", + "d97b429aa6bd4e939cf89819ef840012", + "81d71923e94545878510e6d96f866b0f", + "c390ba7fad2f4788aad5f0940bd379e5", + "df9720767bd947c69ca5d72573159231", + "1759fbfb1e4246ce811d60ca40629d88", + "27016c45bd2243098ec392ad4849a05d", + "dd4f12a02c154d788901054c65259844", + "29f99325a510450199095285baa3d813", + "99272e059aaa41aca098e57455075b58", + "bd474430a72341fb8b36d39d54bbcc66", + "865c4fe0f5d545f6a105532a707c801e", + "2a2c9d74cd9b44289821798baa4a9975", + "d598a363048644cd96fa2036b37b3508", + "85c1aa2c5d6f4628b705a19d8751e64c", + "377d10bd4eda48fd9821f24db6d44c0b", + "1a8a40c8074f4edab43ea7cfc7606d55", + "b866371e1e974218ac03b641219b1dab", + "f4e99eddb6ee4329a2fa803f48c9f01c", + "3821ba5ff2074c9f9101ad45c7eb26d4", + "ba01810cc262488ca04cd270cbd18cc6", + "116cfb137a804861b88448a7c3efa1d0", + "0f11aa47a7f04f5cb22fdf535c0e13be", + "24f8ad3fdc8549f69ff7af7516c586d4", + "3842c82332b7496a9686ee4b90ca0a9c", + "f2ed661079f748d288f3d359187abc7e", + "fe5a66a1682e42018c61351bc00d6a90", + "400656299c294b389d452413d1909f97", + "6e705e9a55674e86b2b9276faa782a66" + ] + }, + "id": "RK8KQ6g_n9a7", + "outputId": "52f96443-5719-41dc-dea4-c7d9256ba2f4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Launching LLMfx process\n", + "step - \t1 - \tcreating object - ready to start processing.\n", + "step - \t2 - \tloading tool - sentiment\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5ff40e631f664081a8320d233f30c278", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00=1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.34.129)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.3)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.7.3)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.2)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.129 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.34.129)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (0.10.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.129->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.129->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.129->boto3>=1.24.53->llmware) (1.16.0)\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "from llmware.models import ModelCatalog\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "erpOjVoUGZPU" + }, + "source": [ + "To add more strings to analyze, add them to Text_to_analyze.\n", + "To add more queries, add them to queries_list.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "id": "Ls6GHIEqF1uq" + }, + "outputs": [], + "source": [ + "Text_to_analyze = [\"I am John Doe, I visited Peter, a therapist in the year 2019. I travelled to NYC for around 5 therapy sessions, I was advised to be more outgoing.\"]\n", + "queries_list = [\"consultant specialty\", \"Consultant name\", \"Consultant location\", \"Consultant Advice\"]\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aWw7uro5C1ym", + "outputId": "b33fbf63-171c-4b5b-f33a-b102282b42d2" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Query 1: consultant specialty\n", + "extract response: {'consultant_specialty': ['Therapist']}\n", + "Query 2: Consultant name\n", + "extract response: {'Consultant_name': ['Peter']}\n", + "Query 3: Consultant location\n", + "extract response: {'Consultant_location': ['NYC']}\n", + "Query 4: Consultant Advice\n", + "extract response: {'Consultant_Advice': ['Be more outgoing.']}\n", + "output dictionary: \n", + "{'consultant_specialty': ['Therapist'], 'Consultant_name': ['Peter'], 'Consultant_location': ['NYC'], 'Consultant_Advice': ['Be more outgoing.']}\n" + ] + } + ], + "source": [ + "# load the model\n", + "model = ModelCatalog().load_model(\"slim-extract-tool\",sample=False, temperature=0.0, max_output=200)\n", + "# loop through the text and queries, calling the model each time\n", + "output_dict = {}\n", + "for i, sample in enumerate(Text_to_analyze):\n", + " for j, query in enumerate(queries_list):\n", + " print(f\"Query {j+1}: {query}\")\n", + " response = model.function_call(sample, function=\"extract\", params=[query])\n", + " output_dict.update(response[\"llm_response\"])\n", + " if response[\"llm_response\"] == []:\n", + " print(\"No response\")\n", + " print(\"extract response: \", response[\"llm_response\"])\n", + "\n", + "\n", + " # display the response on the screen\n", + "print(\"output dictionary: \" )\n", + "print(output_dict)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/NoteBook_Examples/text2sql-end-to-end-2-notebook.ipynb b/tutorials/notebooks/NoteBook_Examples/text2sql-end-to-end-2-notebook.ipynb new file mode 100644 index 0000000..ec3024e --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/text2sql-end-to-end-2-notebook.ipynb @@ -0,0 +1,557 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Natural Language Queries for SQL using SLIM (Full End-to-End Example)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You've might have heard of SQL. It's a widely used programming language for storing and processing information in relational databases - simply put, relational databases store data in tables, where each row stores an entity and each column stores an attribute for that entity.\n", + "\n", + "Let's say we have a table called customers in a relational database. If I wanted to access the names of all customers (customer_names) that have an annual_spend of at least $1000, I would have to formulate an SQL query like this:\n", + "```SQL\n", + "SELECT customer_names FROM customers WHERE annual_spend >= 1000\n", + "```\n", + "I would then run this query against the database to access my results.\n", + "\n", + "But what if AI 🤖 could do all this for us?\n", + "\n", + "LLMWare allows us to do just that, making use of small language models. These are models of a smaller scale, with fewer and less precise parameters that don't require much computational power to run. Their main advantage is that they run locally on a CPU, without an internet connection or a GPU. This enables use to make our queries entirely in natural language and still get accurate results!\n", + "\n", + "Let's look at an example of how to do this from start to finish." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### For Google Colab users" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are using Colab for free, we highly recommend you activate the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as apposed to the ~13GB Colab gives automatically.\n", + "\n", + "To activate T4:\n", + "1. click on the \"Runtime\" tab\n", + "2. click on \"Change runtime type\"\n", + "3. select T4 GPU under Hardware Accelerator\n", + "\n", + "NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Installing and importing dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Defaulting to user installation because normal site-packages is not writeable\n", + "Requirement already satisfied: llmware in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (0.2.14)\n", + "Requirement already satisfied: boto3==1.24.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.24.53)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.19.4)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.26.0)\n", + "Requirement already satisfied: openai>=1.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.13.3)\n", + "Requirement already satisfied: pymongo>=4.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.7.2)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.15.2)\n", + "Requirement already satisfied: torch>=1.13.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (2.1.0+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.38.2)\n", + "Requirement already satisfied: Wikipedia-API==0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.6.0)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: einops==0.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.7.0)\n", + "Requirement already satisfied: librosa>=0.10.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: botocore<1.28.0,>=1.27.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (1.27.96)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3==1.24.53->llmware) (0.6.2)\n", + "Requirement already satisfied: typing-extensions>=4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (4.8.0)\n", + "Requirement already satisfied: tzdata in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (2023.3)\n", + "Requirement already satisfied: requests in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from Wikipedia-API==0.6.0->llmware) (2.31.0)\n", + "Requirement already satisfied: filelock in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (3.13.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (2023.10.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (4.66.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (23.1)\n", + "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.11.3)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.1)\n", + "Requirement already satisfied: joblib>=0.14 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (5.1.1)\n", + "Requirement already satisfied: numba>=0.51.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.59.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (4.0.0)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (1.9.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (0.23.3)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (2.6.4)\n", + "Requirement already satisfied: sniffio in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from openai>=1.0.0->llmware) (1.3.0)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: sympy in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (3.2.1)\n", + "Requirement already satisfied: jinja2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from torch>=1.13.1->llmware) (3.1.2)\n", + "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from transformers>=4.36.0->llmware) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from transformers>=4.36.0->llmware) (0.4.2)\n", + "Requirement already satisfied: idna>=2.8 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (2.10)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.26.18)\n", + "Requirement already satisfied: certifi in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2023.7.22)\n", + "Requirement already satisfied: httpcore<0.17.0,>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.16.3)\n", + "Requirement already satisfied: rfc3986<2,>=1.3 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from rfc3986[idna2008]<2,>=1.3->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (1.5.0)\n", + "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.42.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (3.10.0)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.16.3 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.16.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.2.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.2.0)\n", + "Requirement already satisfied: cffi>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.15.1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from jinja2->torch>=1.13.1->llmware) (2.1.3)\n", + "Requirement already satisfied: mpmath>=0.19 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.21)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from httpcore<0.17.0,>=0.15.0->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.14.0)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from llmware.agents import SQLTables, LLMfx\n", + "from llmware.models import ModelCatalog\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Worker function" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The process to generate an output for the given input query is as follows:\n", + "1. If `create_new_table` is set to `True`, then:\n", + " - Check to see if the SLIM SQL tool is already downloaded, if not, download it using the `ModelCatalog` class\n", + " - Load in the sample CSV file `customer_table.csv`\n", + " - Create an SQL table from the CSV file using the `SQLTables` class\n", + "2. Create an agent, an instance of the `LLMfx` class, and load in the SQL tool\n", + "3. For each query in the `query_list`, call the `query_db` function on the agent, which:\n", + " - looks up the table schema in the db using the table_name\n", + " - packages the text-2-sql query prompt\n", + " - executes sql method to convert the prompt into a sql query\n", + " - attempts to execute the sql query on the db\n", + " - returns the db results as 'research' output\n", + "4. Print the result from the `agent`'s `research_list`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def sql_e2e_test_script(table_name=\"customers1\",create_new_table=False):\n", + "\n", + " \"\"\" This is the end-to-end execution script. \"\"\"\n", + "\n", + " # create table if needed to set up\n", + " if create_new_table:\n", + "\n", + " # looks to pull sample csv 'customer_table.csv' from slim-sql-tool model package files\n", + " sql_tool_repo_path = os.path.join(LLMWareConfig().get_model_repo_path(), \"slim-sql-tool\")\n", + "\n", + " if not os.path.exists(sql_tool_repo_path):\n", + " ModelCatalog().load_model(\"llmware/slim-sql-tool\")\n", + "\n", + " files = os.listdir(sql_tool_repo_path)\n", + " csv_file = \"customer_table.csv\"\n", + "\n", + " if csv_file in files:\n", + "\n", + " # to create a testing table from a csv\n", + " sql_db = SQLTables(experimental=True)\n", + " sql_db.create_new_table_from_csv(sql_tool_repo_path, csv_file, table_name=table_name)\n", + " # end - creating table\n", + "\n", + " print(\"update: successfully created new db table\")\n", + " else:\n", + " print(\"something has gone wrong - could not find customer_table.csv inside the slim-sql-tool file package\")\n", + "\n", + " # query starts here\n", + " agent = LLMfx()\n", + " agent.load_tool(\"sql\", sample=False, get_logits=True, temperature=0.0)\n", + "\n", + " # Pass direct queries to the DB\n", + "\n", + " query_list = [\"Which customers have vip customer status of yes?\",\n", + " \"What is the highest annual spend of any customer?\",\n", + " \"Which customer has account number 1234953\",\n", + " \"Which customer has the lowest annual spend?\",\n", + " \"Is Susan Soinsin a vip customer?\"]\n", + "\n", + " for i, query in enumerate(query_list):\n", + "\n", + " # query_db method is doing all of the work\n", + " # -- looks up the table schema in the db using the table_name\n", + " # -- packages the text-2-sql query prompt\n", + " # -- executes sql method to convert the prompt into a sql query\n", + " # -- attempts to execute the sql query on the db\n", + " # -- returns the db results as 'research' output\n", + "\n", + " response = agent.query_db(query, table=table_name)\n", + "\n", + " for x in range(0,len(agent.research_list)):\n", + " print(\"research: \", x, agent.research_list[x])\n", + "\n", + " return 0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to delete a table in the experimental database, you can use the `delete_table()` function." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def delete_table(table_name):\n", + "\n", + " \"\"\" Start fresh in testing - delete table in experimental local SQLite DB \"\"\"\n", + "\n", + " sql_db = SQLTables(experimental=True)\n", + " sql_db.delete_table(table_name, confirm_delete=True)\n", + "\n", + " return True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to delete the entire database, you use the `delete_db()` function." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "def delete_db():\n", + "\n", + " \"\"\" Start fresh in testing - deletes SQLite DB and starts over. \"\"\"\n", + "\n", + " sql_db = SQLTables(experimental=True)\n", + " sql_db.delete_experimental_db(confirm_delete=True)\n", + "\n", + " return True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Main block" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now run our code by calling the `sql_e2e_test_script` function." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2d9d5f328d9b4b0f8bac965385d61d89", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 6 files: 0%| | 0/6 [00:00=1.24.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.24.53)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.19.4)\n", + "Requirement already satisfied: numpy>=1.23.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (1.26.0)\n", + "Requirement already satisfied: pymongo>=4.7.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (4.7.2)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.15.2)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (4.8.0)\n", + "Requirement already satisfied: tzdata in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from psycopg==3.1.17->llmware) (2023.3)\n", + "Requirement already satisfied: botocore<1.28.0,>=1.27.53 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3>=1.24.53->llmware) (1.27.96)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from boto3>=1.24.53->llmware) (0.6.2)\n", + "Requirement already satisfied: filelock in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (3.13.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (2023.10.0)\n", + "Requirement already satisfied: requests in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (4.66.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from huggingface-hub>=0.19.4->llmware) (23.1)\n", + "Requirement already satisfied: audioread>=2.1.9 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.11.3)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.1)\n", + "Requirement already satisfied: joblib>=0.14 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.3.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (5.1.1)\n", + "Requirement already satisfied: numba>=0.51.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.59.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from botocore<1.28.0,>=1.27.53->boto3>=1.24.53->llmware) (1.26.18)\n", + "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.42.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (3.10.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from requests->huggingface-hub>=0.19.4->llmware) (2.10)\n", + "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from requests->huggingface-hub>=0.19.4->llmware) (2023.7.22)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.2.0)\n", + "Requirement already satisfied: cffi>=1.0 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.15.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.21)\n", + "Requirement already satisfied: six>=1.5 in c:\\users\\prash\\appdata\\roaming\\python\\python311\\site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3>=1.24.53->llmware) (1.16.0)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n", + "[notice] A new release of pip is available: 24.0 -> 24.1.2\n", + "[notice] To update, run: python.exe -m pip install --upgrade pip\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from llmware.models import ModelCatalog" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generating questions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this function, we will see how we can generate questions about the `source_passage` using the slim-q-gen-tiny-tool. The workflow for this function is as follows:\n", + "- Load in the the model using `ModelCatalog`\n", + "- Use `function_call()` to generate questions from the model `number_of_tries` times\n", + "- Add the generated question to the `questions` list only if it is unique and is not already in the list\n", + "- Output the `questions`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def hello_world_test(source_passage, q_model=\"slim-q-gen-tiny-tool\", number_of_tries=10, question_type=\"question\", temperature=0.5):\n", + "\n", + " \"\"\" Shows a basic example of generating questions from a text passage, running a number of inferences,\n", + " and then keeping only the unique questions generated.\n", + "\n", + " -- source_passage = text passage\n", + " -- number_of_tries = integer number of times to call the model to generate a question\n", + " -- question_type = \"question\" | \"boolean\" | \"multiple choice\"\n", + "\n", + " \"\"\"\n", + "\n", + " # recommend using temperature of 0.2 - 0.8 - for multiple choice, use lower end of the range\n", + " q_model = ModelCatalog().load_model(q_model, sample=True, temperature=temperature)\n", + "\n", + " questions = []\n", + "\n", + " for x in range(0, number_of_tries):\n", + "\n", + " response = q_model.function_call(source_passage, params=[question_type], get_logits=False)\n", + "\n", + " # expect response in the form of: \"llm_response\": {\"question\": [\"generated question?\"] }\n", + "\n", + " if response:\n", + " if \"llm_response\" in response:\n", + " if \"question\" in response[\"llm_response\"]:\n", + " new_q = response[\"llm_response\"][\"question\"]\n", + "\n", + " # keep only new questions\n", + " if new_q not in questions:\n", + " questions.append(new_q)\n", + "\n", + " print(f\"inference {x} - response: {response['llm_response']}\")\n", + "\n", + " print(f\"\\nDe-duped list of questions created\\n\")\n", + " for i, question in enumerate(questions):\n", + "\n", + " print(f\"new generated questions: {i} - {question}\")\n", + "\n", + " return questions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Dueling AIs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This function will allow us to generate the questions using the same process as above, but then have a different model answer those questions. The process is as follows:\n", + "- Generate the `questions` list using the same steps as the previous function.\n", + "- Load in the answer model using `ModelCatalog`\n", + "- Answer each question in `questions` using the `inference()` function of the answer model." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def ask_and_answer_game(source_passage, q_model=\"slim-q-gen-tiny-tool\", number_of_tries=10, question_type=\"question\",\n", + " temperature=0.5):\n", + "\n", + " \"\"\" Shows a simple two model game of using q-gen model to generate a question, and then a second model\n", + " to answer the question generated. \"\"\"\n", + "\n", + " # this is the model that will generate the 'question'\n", + " q_model = ModelCatalog().load_model(q_model, sample=True, temperature=temperature)\n", + "\n", + " # this will be the model used to 'answer' the question\n", + " answer_model = ModelCatalog().load_model(\"bling-phi-3-gguf\")\n", + "\n", + " questions = []\n", + "\n", + " print(f\"\\nGenerating a set of questions automatically from the source passage.\\n\")\n", + "\n", + " for x in range(0,number_of_tries):\n", + "\n", + " response = q_model.function_call(source_passage, params=[question_type], get_logits=False)\n", + "\n", + " if response:\n", + " if \"llm_response\" in response:\n", + " if \"question\" in response[\"llm_response\"]:\n", + " new_q = response[\"llm_response\"][\"question\"]\n", + "\n", + " # only keep new questions\n", + " if new_q and new_q not in questions:\n", + " questions.append(new_q)\n", + "\n", + " print(f\"inference - {x} - response: {response['llm_response']}\")\n", + "\n", + " print(\"\\nAnswering the generated questions\\n\")\n", + " for i, question in enumerate(questions):\n", + "\n", + " print(f\"\\nquestion: {i} - {question}\")\n", + " if isinstance(question, list) and len(question) > 0:\n", + " response = answer_model.inference(question[0], add_context=source_passage)\n", + " print(f\"response: \", response[\"llm_response\"])\n", + "\n", + " return True" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Main block" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we state our source text and call both functions above." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# test passage pulled from CNBC news story on Tuesday, May 28, 2024\n", + "test_passage = (\"OpenAI said Tuesday it has established a new committee to make recommendations to the \"\n", + " \"company’s board about safety and security, weeks after dissolving a team focused on AI safety. \"\n", + " \"In a blog post, OpenAI said the new committee would be led by CEO Sam Altman as well as \"\n", + " \"Bret Taylor, the company’s board chair, and board member Nicole Seligman. The announcement \"\n", + " \"follows the high-profile exit this month of an OpenAI executive focused on safety, \"\n", + " \"Jan Leike. Leike resigned from OpenAI leveling criticisms that the company had \"\n", + " \"under-invested in AI safety work and that tensions with OpenAI’s leadership had \"\n", + " \"reached a breaking point.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "inference 0 - response: {'question': ['Who are the two members of the new safety and security committee?']}\n", + "inference 1 - response: {'question': ['What is the name of the new committee?']}\n", + "inference 2 - response: {'question': ['What is the name of the executive who resigned?']}\n", + "inference 3 - response: {'question': ['What is the name of the CEO of the company?']}\n", + "inference 4 - response: {'question': ['What is the name of the board chair?']}\n", + "inference 5 - response: {'question': ['Who is the chairman of OpenAI?']}\n", + "inference 6 - response: {'question': ['What is a list of the key points in the announcement?']}\n", + "inference 7 - response: {'question': ['What is a list of three key points?']}\n", + "inference 8 - response: {'question': ['What is the name of the executive who resigned?']}\n", + "inference 9 - response: {'question': ['What is the name of the executive who resigned from OpenAI?']}\n", + "\n", + "De-duped list of questions created\n", + "\n", + "new generated questions: 0 - ['Who are the two members of the new safety and security committee?']\n", + "new generated questions: 1 - ['What is the name of the new committee?']\n", + "new generated questions: 2 - ['What is the name of the executive who resigned?']\n", + "new generated questions: 3 - ['What is the name of the CEO of the company?']\n", + "new generated questions: 4 - ['What is the name of the board chair?']\n", + "new generated questions: 5 - ['Who is the chairman of OpenAI?']\n", + "new generated questions: 6 - ['What is a list of the key points in the announcement?']\n", + "new generated questions: 7 - ['What is a list of three key points?']\n", + "new generated questions: 8 - ['What is the name of the executive who resigned from OpenAI?']\n" + ] + }, + { + "data": { + "text/plain": [ + "[['Who are the two members of the new safety and security committee?'],\n", + " ['What is the name of the new committee?'],\n", + " ['What is the name of the executive who resigned?'],\n", + " ['What is the name of the CEO of the company?'],\n", + " ['What is the name of the board chair?'],\n", + " ['Who is the chairman of OpenAI?'],\n", + " ['What is a list of the key points in the announcement?'],\n", + " ['What is a list of three key points?'],\n", + " ['What is the name of the executive who resigned from OpenAI?']]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hello_world_test(test_passage,q_model=\"slim-q-gen-tiny-tool\",number_of_tries=10, question_type=\"question\", temperature=0.5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see each question that was generated `number_of_tries` times, and then the final question list with the duplicates removed." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Generating a set of questions automatically from the source passage.\n", + "\n", + "inference - 0 - response: {'question': ['What is the name of the executive who resigned?']}\n", + "inference - 1 - response: {'question': ['When was this announcement made?']}\n", + "inference - 2 - response: {'question': ['What is the name of one of the members of the new committee?']}\n", + "inference - 3 - response: {'question': ['What is one of the names of the people who will lead the new committee?']}\n", + "inference - 4 - response: {'question': ['What is the name of the person who will lead the new committee?']}\n", + "inference - 5 - response: {'question': ['Who is leading the new advisory group?']}\n", + "inference - 6 - response: {'question': ['What is the name of the executive who resigned?']}\n", + "inference - 7 - response: {'question': ['What is the name of the person who is leading the new committee?']}\n", + "inference - 8 - response: {'question': ['What is one role of Nicole Seligman?']}\n", + "inference - 9 - response: {'question': ['What is one of the names of one of the members of this new committee?']}\n", + "\n", + "Answering the generated questions\n", + "\n", + "\n", + "question: 0 - ['What is the name of the executive who resigned?']\n", + "response: Jan Leike\n", + "\n", + "question: 1 - ['When was this announcement made?']\n", + "response: Tuesday\n", + "\n", + "question: 2 - ['What is the name of one of the members of the new committee?']\n", + "response: Bret Taylor\n", + "\n", + "question: 3 - ['What is one of the names of the people who will lead the new committee?']\n", + "response: Sam Altman\n", + "\n", + "question: 4 - ['What is the name of the person who will lead the new committee?']\n", + "response: Sam Altman\n", + "\n", + "question: 5 - ['Who is leading the new advisory group?']\n", + "response: CEO Sam Altman\n", + "\n", + "question: 6 - ['What is the name of the person who is leading the new committee?']\n", + "response: Sam Altman\n", + "\n", + "question: 7 - ['What is one role of Nicole Seligman?']\n", + "response: Board Chair of OpenAI\n", + "\n", + "question: 8 - ['What is one of the names of one of the members of this new committee?']\n", + "response: Bret Taylor\n" + ] + }, + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ask_and_answer_game(test_passage,q_model=\"slim-q-gen-phi-3-tool\", number_of_tries=10, question_type=\"question\", temperature=0.5)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here, we can see each question generated, then the responses to each unique (non-duplicate) question." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tutorials/notebooks/NoteBook_Examples/whispercpp_nb.ipynb b/tutorials/notebooks/NoteBook_Examples/whispercpp_nb.ipynb new file mode 100644 index 0000000..31e544f --- /dev/null +++ b/tutorials/notebooks/NoteBook_Examples/whispercpp_nb.ipynb @@ -0,0 +1,387 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "jdUlCWCXWX-C" + }, + "source": [ + "This example shows how to use llmware provided sample files for testing with WhisperCPP, integrated as of\n", + " llmware 0.2.11.\n", + "\n", + " # examples - \"famous_quotes\" | \"greatest_speeches\" | \"youtube_demos\" | \"earnings_calls\"\n", + "\n", + " -- famous_quotes - approximately 20 small .wav files with clips from old movies and speeches\n", + " -- greatest_speeches - approximately 60 famous historical speeches in english\n", + " -- youtube_videos - wav files of ~3 llmware youtube videos\n", + " -- earnings_calls - wav files of ~4 public company earnings calls (gathered from public investor relations)\n", + "\n", + " These sample files are hosted in a non-restricted AWS S3 bucket, and downloaded via the Setup method\n", + " `load_sample_voice_files`. There are two options:\n", + "\n", + " -- small_only = True: only pulls the 'famous_quotes' samples\n", + " -- small_only = False: pulls all of the samples (requires ~1.9 GB in total)\n", + "\n", + " Please note that all of these samples have been pulled from open public domain sources, including the\n", + " Internet Archives, e.g., https://archive.org. These sample files are being provided solely for the purpose of\n", + " testing the code scripts below. Please do not use them for any other purpose.\n", + "\n", + " To run these examples, please make sure to `pip install librosa`\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WM0bYHGyWpGu" + }, + "source": [ + "# If you are using Colab for free, we highly recommend you activate the T4 GPU\n", + "# hardware accelerator. Our models are designed to run with at least 16GB\n", + "# of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed\n", + "# to the ~13GB Colab gives automatically.\n", + "# To activate T4:\n", + "# 1. click on the \"Runtime\" tab\n", + "# 2. click on \"Change runtime type\"\n", + "# 3. select T4 GPU under Hardware Accelerator\n", + "# NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4vi_LENqVGHp", + "outputId": "4b296c7b-e67a-4dee-faad-da3636c79858" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: llmware in /usr/local/lib/python3.10/dist-packages (0.3.0)\n", + "Requirement already satisfied: boto3>=1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.34.129)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.3)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.7.3)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.2)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.129 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.34.129)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (0.10.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.129->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.129->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.129->boto3>=1.24.53->llmware) (1.16.0)\n", + "Requirement already satisfied: librosa in /usr/local/lib/python3.10/dist-packages (0.10.2.post1)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa) (3.0.1)\n", + "Requirement already satisfied: numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.25.2)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.3.7)\n", + "Requirement already satisfied: typing-extensions>=4.1.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (4.12.2)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.0.8)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from lazy-loader>=0.1->librosa) (24.1)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa) (4.2.2)\n", + "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa) (2.31.0)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa) (2.22)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->pooch>=1.1->librosa) (2024.6.2)\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install librosa" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "YlMbZEubW3P7" + }, + "source": [ + "\n", + "**Set the verbose config to TRUE if you would like a more detailed readout of the results.**\n", + "\n", + "**Set your language, English (en) is default.**" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "baMdUNuIwlrm" + }, + "outputs": [], + "source": [ + "\n", + "\n", + "import os\n", + "from llmware.models import ModelCatalog\n", + "from llmware.gguf_configs import GGUFConfigs\n", + "from llmware.setup import Setup\n", + "\n", + "# optional / to adjust various parameters of the model\n", + "GGUFConfigs().set_config(\"whisper_cpp_verbose\", \"OFF\")\n", + "GGUFConfigs().set_config(\"whisper_cpp_realtime_display\", True)\n", + "\n", + "# note: english is default output - change to 'es' | 'fr' | 'de' | 'it' ...\n", + "GGUFConfigs().set_config(\"whisper_language\", \"en\")\n", + "GGUFConfigs().set_config(\"whisper_remove_segment_markers\", True)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "cWTYJLfXWOzH" + }, + "outputs": [], + "source": [ + "def sample_files(example=\"famous_quotes\", small_only=False):\n", + "\n", + " \"\"\" Execute a basic inference on Voice-to-Text model passing a file_path string \"\"\"\n", + "\n", + " voice_samples = Setup().load_voice_sample_files(small_only=small_only)\n", + "\n", + " examples = [\"famous_quotes\", \"greatest_speeches\", \"youtube_demos\", \"earnings_calls\"]\n", + "\n", + " if example not in examples:\n", + " print(\"choose one of the following - \", examples)\n", + " return 0\n", + "\n", + " fp = os.path.join(voice_samples,example)\n", + "\n", + " files = os.listdir(fp)\n", + "\n", + " # these are the two key lines\n", + " whisper_base_english = \"whisper-cpp-base-english\"\n", + "\n", + " model = ModelCatalog().load_model(whisper_base_english)\n", + "\n", + " for f in files:\n", + "\n", + " if f.endswith(\".wav\"):\n", + "\n", + " prompt = os.path.join(fp,f)\n", + "\n", + " print(f\"\\n\\nPROCESSING: prompt = {prompt}\")\n", + "\n", + " response = model.inference(prompt)\n", + "\n", + " print(\"\\nllm response: \", response[\"llm_response\"])\n", + " print(\"usage: \", response[\"usage\"])\n", + "\n", + " return 0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fAf9KlzBXhSm" + }, + "source": [ + "**Change the audio you want transcribed from our collection of sample files, or alter the code to point towards your own audio files.**" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "vo7_pq32WMz7", + "outputId": "aae49981-9fda-4374-c42a-c9610bcbb4b3" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/cusack_say_anything_career.wav\n", + "\n", + "llm response: i want to sell anything by anything or process anything as a career i want to sell anything but a process to buy anything so the process process anything so bought or process repair anything so bought a process you know it's a career i want to\n", + "usage: {'duration-seconds': 16.8298125, 'segments': 4, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/all_presidents_money.wav\n", + "\n", + "llm response: supposedly he's got a lawyer with twenty five thousand dollars and a brown paper bag. \"Hey follow the money.\" What do you mean? Well, I can't tell you that. But you could tell me that. No, I have to do this my way. You tell me what you know and I'll confirm. I'll keep you in the right direction if I can, but that's all. Just follow the money.\n", + "usage: {'duration-seconds': 31.0826875, 'segments': 9, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/cant_handle.wav\n", + "\n", + "llm response: I said I want the truth. You can't handle the truth. I said I want the truth. You can't handle the truth.\n", + "usage: {'duration-seconds': 7.096625, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/none_shall_pass.wav\n", + "\n", + "llm response: None shall pass. What none shall pass?\n", + "usage: {'duration-seconds': 4.8298125, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/field_dreams_he_will_come.wav\n", + "\n", + "llm response: if you build it, he will come.\n", + "usage: {'duration-seconds': 3.2624375, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/usual_suspects_smarter2.wav\n", + "\n", + "llm response: Let me get right to the point. I'm smarter than you. And I'm going to find out what I want to know. And I'm going to get it from you whether you like it or not.\n", + "usage: {'duration-seconds': 6.77225, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/life_moves_fast.wav\n", + "\n", + "llm response: life moves pretty fast you don't stop and look around once in a while\n", + "usage: {'duration-seconds': 4.5989375, 'segments': 2, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/bonds.wav\n", + "\n", + "llm response: Bond. James Bond. My name's Bond. James Bond. My name is Bond. James Bond. James Bond.\n", + "usage: {'duration-seconds': 12.57825, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/willy_wonka_questions.wav\n", + "\n", + "llm response: I'm sorry, but all questions must be submitted in writing.\n", + "usage: {'duration-seconds': 2.66875, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/wall_street_greed.wav\n", + "\n", + "llm response: Greed is good. Greed is right. Greed works.\n", + "usage: {'duration-seconds': 5.7818125, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/optimistic.wav\n", + "\n", + "llm response: The emperor does not share your optimistic appraisal of the situation.\n", + "usage: {'duration-seconds': 4.6734375, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/bond_james_bond.wav\n", + "\n", + "llm response: I admire your luck, Mr. Bond. Jamie's Bond.\n", + "usage: {'duration-seconds': 7.6161875, 'segments': 2, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/t3_judgment_day.wav\n", + "\n", + "llm response: the end of the world is today three hours from now two hours and fifty-three minutes.\n", + "usage: {'duration-seconds': 12.201375, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/t1_be_back.wav\n", + "\n", + "llm response: i'll be back.\n", + "usage: {'duration-seconds': 1.053375, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/jfk.wav\n", + "\n", + "llm response: And so my fellow Americans, ask not what your country can do for you, ask what you can do for your country.\n", + "usage: {'duration-seconds': 11.0, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/apologize.wav\n", + "\n", + "llm response: Alright, alright, I apologize. I'm really, really sorry. I apologize unreservedly. I offer a complete and utter retraction. The imputation was totally without basis in fact and was in no way fair comment and was motivated purely by malice. And I deeply regret any distress that my comments may have caused you or your family, and I hereby undertake not to repeat any such slander at any time in the future.<|endoftext|>\n", + "usage: {'duration-seconds': 28.143, 'segments': 1, 'language': 'en'}\n", + "\n", + "\n", + "PROCESSING: prompt = /root/llmware_data/voice_sample_files/famous_quotes/indiana_last_situation.wav\n", + "\n", + "llm response: Our situation has not improved.\n", + "usage: {'duration-seconds': 2.249625, 'segments': 1, 'language': 'en'}\n" + ] + } + ], + "source": [ + "if __name__ == \"__main__\":\n", + "\n", + " # pick among the four examples: famous_quotes | greatest_speeches | youtube_demos | earnings_calls\n", + "\n", + " sample_files(example=\"famous_quotes\", small_only=False)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/README.md b/tutorials/notebooks/README.md new file mode 100644 index 0000000..881851d --- /dev/null +++ b/tutorials/notebooks/README.md @@ -0,0 +1,47 @@ +# Understanding Python notebooks + +Welcome to our project documentation! A common point of confusion among developers new to data science and machine learning workflows is the relationship and differences between popular Python notebooks, including Google Colab and Jupyter Notebooks. This README aims to clarify these parts to ensure everyone is on the same page. + +## What are Jupyter Notebooks? + +Jupyter Notebooks is an open-source web application that lets you create and share documents that have live code, equations, visualizations, and narrative text. Uses include: data cleaning and transformation, numerical simulation, statistical modeling, data visualization, machine learning, and much more. + +## What is Google Colab? + +Google Colab (or Colaboratory) is a free Jupyter notebook environment that requires no setup and runs in the cloud. It offers a similar interface to Jupyter Notebooks and lets users write and execute Python in a web browser. Google Colab also provides free access to computing resources, including GPUs and TPUs, making it highly popular for machine learning and data analysis projects. + +## What are marimo notebooks? + +[marimo](https://github.com/marimo-team) is a new open-source Python notebook +that is an alternative to Jupyter notebooks. Unlike Jupyter notebooks, marimo +has interactive UI elements built-in, letting you transform dataframes and +otherwise explore data in a low-code environment. Unlike Jupyter, marimo +notebooks are reusable artifacts: they can be deployed as interactive web apps +and even executed as scripts. + +You can learn more by playing with an [online interactive tutorial](https://marimo.app/?slug=c7h6pz). + + +## Key Similarities + +- **Interface:** All three platforms use a similar user interface, which supports mixing executable code, equations, visualizations, and narrative text in a single document. +- **Language Support:** Primarily, all three notebooks are used for executing Python code. However, Jupyter Notebooks support other languages such as R and Julia. +- **Use Cases:** They are widely used for data analysis, machine learning, and education, allowing for easy sharing of results and methodologies. Additionally, marimo lets data scientists build and share interactive web apps, without requiring any familiarity with frontend development. + +## Key Differences + +- **Execution Environment:** Jupyter and marimo notebooks can be run locally on your machine or on a server, but Google Colab is hosted exclusively in the cloud. +- **Execution Style.** Jupyter notebooks and Google Colab have no understanding of the code in your notebook, and when you run +one cell you must manually run affected cells. marimo notebooks understand the relationship between code cells, similar to a spreadsheet, and can automatically update outputs when a cell is run. +- **Access to Resources:** Google Colab provides free access to hardware accelerators (GPUs and TPUs) which is not inherently available in Jupyter and marimo unless specifically set up by the user on their servers. +- **Collaboration:** Google Colab offers easier collaboration features, similar to Google Docs, letting multiple users work on the same notebook simultaneously. marimo provides an [online playground](https://marimo.app) that runs entirely in your browser. + +## Conclusion + +While Google Colab and Jupyter Notebooks might seem different they are built on the same idea and offer similar functionalities with a few distinctions, mainly in execution environment and access to computing resources. Marimo notebooks feel similar to Jupyter +and Google Colab notebooks, but they can also be used as interactive web apps and reusable Python scripts. + +Understanding these platforms' capabilities can significantly enhance your data science and machine learning projects. + +We hope this guide has helped clarify the similarities and differences between Google Colab and Jupyter Notebooks. Happy coding! + diff --git a/tutorials/notebooks/fast_start_examples/example_1_create_first_library_version_1.ipynb b/tutorials/notebooks/fast_start_examples/example_1_create_first_library_version_1.ipynb new file mode 100644 index 0000000..3d42384 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_1_create_first_library_version_1.ipynb @@ -0,0 +1,357 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "iQqHW_jdrgGa" + }, + "source": [ + "# **If you are using Colab for free, we highly recommend you activate the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed to the 13GB Colab gives automatically.**\n", + "# **To active T4:**\n", + "# **1. click on the \"Runtime\" tab**\n", + "# **2. click on \"Change runtime type\"**\n", + "# **3. select T4 GPU under Hardware Accelerator**\n", + "# **NOTE: there is a weekly usage limit on using T4 for free**\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "h3wwcTnayFTw", + "outputId": "e7dd0212-9398-4ddf-f17a-f5a83f29f40e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.3.0-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.120-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.3/139.3 kB\u001b[0m \u001b[31m698.5 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (669 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m669.1/669.1 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "Collecting botocore<1.35.0,>=1.34.120 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.120-py3-none-any.whl (12.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m31.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.1-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.2/82.2 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.120 botocore-1.34.120 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.0 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.3 s3transfer-0.10.1\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "SmOaSv8_yWDw" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.library import Library\n", + "from llmware.retrieval import Query\n", + "from llmware.setup import Setup\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yBR2N26tyg9E" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "LLMWareConfig().set_config(\"debug_mode\", 2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "kLLaRPK8y92L" + }, + "outputs": [], + "source": [ + "sample_folders = [\"Agreements\", \"Invoices\", \"UN-Resolutions-500\", \"SmallLibrary\", \"FinDocs\", \"AgreementsLarge\"]\n", + "library_name = \"example1_library\"\n", + "selected_folder = sample_folders[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wnGCyOkYze-a", + "outputId": "d9d56aa3-7ec4-4b6b-9513-d13ab537ff09" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Example - Parsing Files into Library\n", + "\n", + "Step 1 - creating library example1_library\n" + ] + } + ], + "source": [ + "print(f\"\\nExample - Parsing Files into Library\")\n", + "print(f\"\\nStep 1 - creating library {library_name}\")\n", + "library = Library().create_new_library(library_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0grz6kFBz4C9", + "outputId": "82530467-780e-42d0-b4dc-7a61977a9f16" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 2 - loading the llmware sample files and saving at: /root/llmware_data/sample_files\n" + ] + } + ], + "source": [ + "sample_files_path = Setup().load_sample_files(over_write=False)\n", + "print(f\"Step 2 - loading the llmware sample files and saving at: {sample_files_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "8jbY2jP_0D-9", + "outputId": "5682b88c-de56-459b-a8fb-a5fb3fe711a6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 3 - parsing and indexing files from /root/llmware_data/sample_files/Agreements\n" + ] + } + ], + "source": [ + "ingestion_folder_path = os.path.join(sample_files_path, selected_folder)\n", + "print(f\"Step 3 - parsing and indexing files from {ingestion_folder_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VW-upUwa0SrA", + "outputId": "4ea25ddd-5318-4b21-f18b-b79fbafb3af0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 4 - completed parsing - {'docs_added': 15, 'blocks_added': 2211, 'images_added': 0, 'pages_added': 204, 'tables_added': 0, 'rejected_files': []}\n" + ] + } + ], + "source": [ + "parsing_output = library.add_files(ingestion_folder_path)\n", + "print(f\"Step 4 - completed parsing - {parsing_output}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "S4Sx-o-s0cj6", + "outputId": "26fc3037-d227-45d8-b253-9a2d7a651300" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 5 - updated library card - documents - 15 - blocks - 2211 - {'_id': 1, 'library_name': 'example1_library', 'embedding': [{'embedding_status': 'no', 'embedding_model': 'none', 'embedding_db': 'none', 'embedded_blocks': 0, 'embedding_dims': 0, 'time_stamp': 'NA'}], 'knowledge_graph': 'no', 'unique_doc_id': 15, 'documents': 15, 'blocks': 2211, 'images': 0, 'pages': 204, 'tables': 0, 'account_name': 'llmware'}\n" + ] + } + ], + "source": [ + "updated_library_card = library.get_library_card()\n", + "doc_count = updated_library_card[\"documents\"]\n", + "block_count = updated_library_card[\"blocks\"]\n", + "print(f\"Step 5 - updated library card - documents - {doc_count} - blocks - {block_count} - {updated_library_card}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "IsZgenlo079L", + "outputId": "2e9aeaad-34f0-4218-8d1e-3e4b5288b218" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 6 - library artifacts - including extracted images - saved at folder path - /root/llmware_data/accounts/llmware/example1_library\n" + ] + } + ], + "source": [ + "library_path = library.library_main_path\n", + "print(f\"Step 6 - library artifacts - including extracted images - saved at folder path - {library_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "szyPVmi81Fh6", + "outputId": "b159172b-b8c1-4f39-bc19-753c8b401580" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Step 7 - running a test query - base salary\n", + "\n", + "query results: 0 {'query': 'base salary', '_id': '1663', 'text': \" Base Salary. For all the services rendered by Executive hereunder, during the Employment Period, Employer shall pay Executive a base salary at the annual rate of $200,000, payable semimonthly in accordance with Employer's normal payroll practices. Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation \", 'doc_ID': 12, 'block_ID': 33, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Metis EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 427, 'coords_y': -1681, 'coords_cx': 2, 'coords_cy': 123, 'external_files': '', 'score': -8.71512754073446, 'similarity': 0.0, 'distance': 0.0, 'matches': [[1, 'base'], [6, 'salary'], [131, 'base'], [136, 'salary'], [265, 'base'], [270, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 1 {'query': 'base salary', '_id': '1807', 'text': \" Base Salary. For all the services rendered by Executive hereunder, during the Employment Period, Employer shall pay Executive a base salary at the annual rate of $4350,000, payable semimonthly in accordance with Employer's normal payroll practices. Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation \", 'doc_ID': 13, 'block_ID': 33, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Rhea EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 427, 'coords_y': -1681, 'coords_cx': 2, 'coords_cy': 122, 'external_files': '', 'score': -8.71512754073446, 'similarity': 0.0, 'distance': 0.0, 'matches': [[1, 'base'], [6, 'salary'], [131, 'base'], [136, 'salary'], [266, 'base'], [271, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 2 {'query': 'base salary', '_id': '2101', 'text': \" Base Salary. For all the services rendered by Executive hereunder, during the Employment Period, Employer shall pay Executive a base salary at the annual rate of $200,000, payable semimonthly in accordance with Employer's normal payroll practices. Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation \", 'doc_ID': 15, 'block_ID': 33, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Persephone EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 427, 'coords_y': -1681, 'coords_cx': 2, 'coords_cy': 122, 'external_files': '', 'score': -8.71512754073446, 'similarity': 0.0, 'distance': 0.0, 'matches': [[1, 'base'], [6, 'salary'], [131, 'base'], [136, 'salary'], [265, 'base'], [270, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 3 {'query': 'base salary', '_id': '22', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 1, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Leto EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 4 {'query': 'base salary', '_id': '166', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 2, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Nyx EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 5 {'query': 'base salary', '_id': '760', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 6, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Gaia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 6 {'query': 'base salary', '_id': '1059', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 8, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Nike EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 7 {'query': 'base salary', '_id': '1203', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 9, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Demeter EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 8 {'query': 'base salary', '_id': '1352', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 10, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Eileithyia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 9 {'query': 'base salary', '_id': '1651', 'text': \"any such event or condition and Employer does not fully correct the situation within thirty (30) days after such notice of Good Reason: (i) a reduction by Employer in Executive's Base Salary (other than a proportional reduction as part of a generalized reduction in the base salaries of senior management of the Company not to exceed 5% of Base Salary then currently in effect); \", 'doc_ID': 12, 'block_ID': 21, 'page_num': 2, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Metis EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Thu Jun 6 13:51:29 2024', 'table': '', 'coords_x': 250, 'coords_y': -1687, 'coords_cx': 1, 'coords_cy': 130, 'external_files': '', 'score': -8.240641465648865, 'similarity': 0.0, 'distance': 0.0, 'matches': [[190, 'base'], [195, 'salary'], [283, 'base'], [355, 'base'], [360, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n" + ] + } + ], + "source": [ + "test_query = \"base salary\"\n", + "print(f\"\\nStep 7 - running a test query - {test_query}\\n\")\n", + "query_results = Query(library).text_query(test_query, result_count=10)\n", + "for i, result in enumerate(query_results):\n", + " text = result[\"text\"]\n", + " file_source = result[\"file_source\"]\n", + " page_number = result[\"page_num\"]\n", + " doc_id = result[\"doc_ID\"]\n", + " block_id = result[\"block_ID\"]\n", + " matches = result[\"matches\"]\n", + " print(\"query results: \", i, result)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_1_create_first_library_version_2.ipynb b/tutorials/notebooks/fast_start_examples/example_1_create_first_library_version_2.ipynb new file mode 100644 index 0000000..72b3d79 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_1_create_first_library_version_2.ipynb @@ -0,0 +1,339 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "3XwVnvz2fJ7i" + }, + "source": [ + "Parsing, Text Chunking and Indexing (Ex. 1): Fast Start to RAG\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "BfZOb3iPduL4", + "outputId": "e49fb246-b37e-47ab-de50-45e1ed9a3cac" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting grpcio<=1.60.0,>=1.49.1\n", + " Downloading grpcio-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m24.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: grpcio\n", + "Successfully installed grpcio-1.60.0\n" + ] + } + ], + "source": [ + "%pip install \"grpcio<=1.60.0,>=1.49.1\" --no-cache-dir ## just to be safe with enviroments\n", + "!pip install -q llmware\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c-irvLXJerCn" + }, + "source": [ + "# ## If deploying locally, use these database options\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "8d4UxQgAe1Ve" + }, + "outputs": [], + "source": [ + "from llmware.configs import LLMWareConfig\n", + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "LLMWareConfig().set_vector_db(\"faiss\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "-AigwWXpfq6i" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.library import Library ### Import for library creating\n", + "from llmware.retrieval import Query ### Import for querying\n", + "from llmware.setup import Setup ### Import for setup\n", + "from llmware.configs import LLMWareConfig ### Import for configs\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "6YYfDA6Wihic", + "outputId": "32d302fa-bdb6-472b-fedd-51c3a9abe521" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Step 1 - creating library example1_library\n" + ] + } + ], + "source": [ + "# Initialize the library name for creating a new library in llmware\n", + "library_name = \"example1_library\"\n", + "print(f\"\\nStep 1 - creating library {library_name}\")\n", + "\n", + "# Create a new library instance with the specified name using llmware's Library class\n", + "library = Library().create_new_library(library_name)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "WX_SA0VdktcY" + }, + "outputs": [], + "source": [ + "sample_folders = [\"Agreements\", \"Invoices\", \"UN-Resolutions-500\", \"SmallLibrary\", \"FinDocs\", \"AgreementsLarge\"]\n", + "selected_folder=sample_folders[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 36 + }, + "id": "7TPf54ecmhLx", + "outputId": "30608dca-a616-43e6-94af-60e39f4d754a" + }, + "outputs": [ + { + "data": { + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "string" + }, + "text/plain": [ + "'Agreements'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "selected_folder" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Xa5PzIxBkciv", + "outputId": "04357372-f0b2-4e28-9812-7087e8cd6c52" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 2 - loading the llmware sample files and saving at: /root/llmware_data/sample_files\n" + ] + } + ], + "source": [ + "sample_files_path = Setup().load_sample_files(over_write=False)\n", + "print (f\"Step 2 - loading the llmware sample files and saving at: {sample_files_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "CqZC-z1aiNET", + "outputId": "04f7a385-81ae-4cdd-ed95-d87588baae93" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 3 - parsing and indexing files from /root/llmware_data/sample_files/Agreements\n" + ] + } + ], + "source": [ + "sample_folder = \"Agreements\" # Example folder\n", + "ingestion_folder_path = os.path.join(sample_files_path, sample_folder)\n", + "print(f\"Step 3 - parsing and indexing files from {ingestion_folder_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "7DCg64fGi8ix", + "outputId": "86f74032-eee2-4c6c-cadd-c256d2d6abbf" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 4 - completed parsing - {'docs_added': 15, 'blocks_added': 1653, 'images_added': 0, 'pages_added': 204, 'tables_added': 0, 'rejected_files': []}\n" + ] + } + ], + "source": [ + "parsing_output = library.add_files(ingestion_folder_path)\n", + "print(f\"Step 4 - completed parsing - {parsing_output}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "toMWVXDtjJ2k", + "outputId": "5128b8d3-6653-4567-f93c-6b6012649a5a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 5 - updated library card - documents - 15 - blocks - 1653 - {'_id': 1, 'library_name': 'example1_library', 'embedding': [{'embedding_status': 'no', 'embedding_model': 'none', 'embedding_db': 'none', 'embedded_blocks': 0, 'embedding_dims': 0, 'time_stamp': 'NA'}], 'knowledge_graph': 'no', 'unique_doc_id': 15, 'documents': 15, 'blocks': 1653, 'images': 0, 'pages': 204, 'tables': 0, 'account_name': 'llmware'}\n" + ] + } + ], + "source": [ + "updated_library_card = library.get_library_card()\n", + "doc_count = updated_library_card[\"documents\"]\n", + "block_count = updated_library_card[\"blocks\"]\n", + "print(f\"Step 5 - updated library card - documents - {doc_count} - blocks - {block_count} - {updated_library_card}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qm5nBqkEhhmx", + "outputId": "2d991e1f-d74e-4cf9-f2a4-820de48ed337" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Step 6 - library artifacts - including extracted images - saved at folder path - /root/llmware_data/accounts/llmware/example1_library\n" + ] + } + ], + "source": [ + "library_path = library.library_main_path\n", + "print(f\"Step 6 - library artifacts - including extracted images - saved at folder path - {library_path}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "gC42gipSj0z2", + "outputId": "d5bdd2ed-33e7-4c55-96fc-18f7b714b169" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Step 7 - running a test query - base salary\n", + "\n", + "query results: 0 {'query': 'base salary', '_id': '25', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 1, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Eileithyia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 134, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 1 {'query': 'base salary', '_id': '250', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 3, 'block_ID': 26, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Bia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1969, 'coords_cx': 2, 'coords_cy': 139, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 2 {'query': 'base salary', '_id': '362', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 4, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Leto EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 134, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 3 {'query': 'base salary', '_id': '469', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 5, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Persephone EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 134, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 4 {'query': 'base salary', '_id': '576', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 6, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Demeter EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 136, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 5 {'query': 'base salary', '_id': '685', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 7, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Nyx EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 134, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 6 {'query': 'base salary', '_id': '794', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 8, 'block_ID': 26, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1969, 'coords_cx': 2, 'coords_cy': 133, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 7 {'query': 'base salary', '_id': '906', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 9, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Rhea EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 134, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 8 {'query': 'base salary', '_id': '1013', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 10, 'block_ID': 24, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Metis EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1825, 'coords_cx': 2, 'coords_cy': 135, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n", + "query results: 9 {'query': 'base salary', '_id': '1122', 'text': \" Executive's base salary shall be reviewed annually by the Board (or the compensation committee of the Board), pursuant to Employer's normal compensation and performance review policies for senior level executives, and may be increased but not decreased. The amount of any increase for each year shall be determined accordingly. For purposes of this Agreement, the term “Base Salary” shall mean the amount of Executive's base salary established from time to time pursuant to this Section 2.2.\", 'doc_ID': 11, 'block_ID': 26, 'page_num': 3, 'content_type': 'text', 'author_or_speaker': '', 'special_field1': '', 'file_source': 'Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'added_to_collection': 'Mon Apr 15 21:18:56 2024', 'table': '', 'coords_x': 1093, 'coords_y': -1969, 'coords_cx': 2, 'coords_cy': 133, 'external_files': '', 'score': -8.027992635076878, 'similarity': 0.0, 'distance': 0.0, 'matches': [[13, 'base'], [18, 'salary'], [379, 'base'], [384, 'salary'], [429, 'base'], [434, 'salary']], 'account_name': 'llmware', 'library_name': 'example1_library'}\n" + ] + } + ], + "source": [ + "test_query = \"base salary\"\n", + "print(f\"\\nStep 7 - running a test query - {test_query}\\n\")\n", + "\n", + "query_results = Query(library).text_query(test_query, result_count=10)\n", + "\n", + "for i, result in enumerate(query_results):\n", + " print(\"query results: \", i, result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "CP2UdFAoiPOf" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "L4", + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_2_build_embeddings_version_1.ipynb b/tutorials/notebooks/fast_start_examples/example_2_build_embeddings_version_1.ipynb new file mode 100644 index 0000000..019e942 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_2_build_embeddings_version_1.ipynb @@ -0,0 +1,583 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "w_r-Acp_4NTz", + "outputId": "d736e906-2ddb-4401-a3bd-31ad7dc218a9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: llmware in /usr/local/lib/python3.10/dist-packages (0.3.0)\n", + "Requirement already satisfied: boto3>=1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.34.120)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.7.3)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.120 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.34.120)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (0.10.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (1.16.0)\n", + "Collecting lancedb\n", + " Downloading lancedb-0.8.2-cp38-abi3-manylinux_2_28_x86_64.whl (19.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m19.1/19.1 MB\u001b[0m \u001b[31m28.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting deprecation (from lancedb)\n", + " Downloading deprecation-2.1.0-py2.py3-none-any.whl (11 kB)\n", + "Collecting pylance==0.12.1 (from lancedb)\n", + " Downloading pylance-0.12.1-cp39-abi3-manylinux_2_28_x86_64.whl (23.1 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.1/23.1 MB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting ratelimiter~=1.0 (from lancedb)\n", + " Downloading ratelimiter-1.2.0.post0-py3-none-any.whl (6.6 kB)\n", + "Requirement already satisfied: requests>=2.31.0 in /usr/local/lib/python3.10/dist-packages (from lancedb) (2.31.0)\n", + "Collecting retry>=0.9.2 (from lancedb)\n", + " Downloading retry-0.9.2-py2.py3-none-any.whl (8.0 kB)\n", + "Requirement already satisfied: tqdm>=4.27.0 in /usr/local/lib/python3.10/dist-packages (from lancedb) (4.66.4)\n", + "Requirement already satisfied: pydantic>=1.10 in /usr/local/lib/python3.10/dist-packages (from lancedb) (2.7.3)\n", + "Requirement already satisfied: attrs>=21.3.0 in /usr/local/lib/python3.10/dist-packages (from lancedb) (23.2.0)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from lancedb) (24.0)\n", + "Requirement already satisfied: cachetools in /usr/local/lib/python3.10/dist-packages (from lancedb) (5.3.3)\n", + "Collecting overrides>=0.7 (from lancedb)\n", + " Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n", + "Requirement already satisfied: pyarrow<15.0.1,>=12 in /usr/local/lib/python3.10/dist-packages (from pylance==0.12.1->lancedb) (14.0.2)\n", + "Requirement already satisfied: numpy>=1.22 in /usr/local/lib/python3.10/dist-packages (from pylance==0.12.1->lancedb) (1.25.2)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->lancedb) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->lancedb) (2.18.4)\n", + "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.10->lancedb) (4.12.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->lancedb) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->lancedb) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->lancedb) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31.0->lancedb) (2024.6.2)\n", + "Requirement already satisfied: decorator>=3.4.2 in /usr/local/lib/python3.10/dist-packages (from retry>=0.9.2->lancedb) (4.4.2)\n", + "Collecting py<2.0.0,>=1.4.26 (from retry>=0.9.2->lancedb)\n", + " Downloading py-1.11.0-py2.py3-none-any.whl (98 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m98.7/98.7 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: ratelimiter, py, overrides, deprecation, retry, pylance, lancedb\n", + "Successfully installed deprecation-2.1.0 lancedb-0.8.2 overrides-7.7.0 py-1.11.0 pylance-0.12.1 ratelimiter-1.2.0.post0 retry-0.9.2\n", + "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.41.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.14.0)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.5.15)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n", + "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (2023.6.0)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (4.12.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.6.2)\n", + "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.3.0+cu121)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.14.0)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.1)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12.1)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch) (8.9.2.26)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /usr/local/lib/python3.10/dist-packages (from torch) (2.20.5)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.3.0)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.5.40)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n", + "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install lancedb\n", + "!pip install transformers\n", + "!pip install torch" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "9ionfrAq4YJ_" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.library import Library\n", + "from llmware.retrieval import Query\n", + "from llmware.setup import Setup\n", + "from llmware.resources import Status\n", + "from llmware.models import ModelCatalog\n", + "from llmware.configs import LLMWareConfig, MilvusConfig\n", + "\n", + "from importlib import util\n", + "\n", + "\n", + "if not util.find_spec(\"torch\") or not util.find_spec(\"transformers\"):\n", + " print(\"\\nto run this example, with the selected embedding model, please install transformers and torch, e.g., \"\n", + " \"\\n`pip install torch`\"\n", + " \"\\n`pip install transformers`\")\n", + "\n", + "if not (util.find_spec(\"chromadb\") or util.find_spec(\"pymilvus\") or util.find_spec(\"lancedb\") or util.find_spec(\"faiss\")):\n", + " print(\"\\nto run this example, you will need to pip install the vector db drivers. see comments above,\")" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "id": "63G2yNI74dCX" + }, + "outputs": [], + "source": [ + "LLMWareConfig.set_active_db(\"sqlite\")\n", + "LLMWareConfig().set_vector_db(\"lancedb\")" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "r-ApXhrB6z57", + "outputId": "6e31467c-2283-4b6d-f724-37c1af942836" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: Creating library: example2_library\n" + ] + } + ], + "source": [ + "print(\"\\nupdate: Creating library: {}\".format(\"example2_library\"))\n", + "library = Library().create_new_library(\"example2_library\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "SNeDdWMa7jHE", + "outputId": "27cbb2fa-cc50-42cb-c328-d96dd7c37023" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embedding record - before embedding [{'embedding_status': 'no', 'embedding_model': 'none', 'embedding_db': 'none', 'embedded_blocks': 0, 'embedding_dims': 0, 'time_stamp': 'NA'}]\n" + ] + } + ], + "source": [ + "embedding_record = library.get_embedding_status()\n", + "print(\"embedding record - before embedding \", embedding_record)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Gosqszd47qRn", + "outputId": "b0117f31-c657-4fa4-f580-f8a9d033c4a2" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Downloading Sample Files\n" + ] + } + ], + "source": [ + "print(\"update: Downloading Sample Files\")\n", + "sample_files_path = Setup().load_sample_files(over_write=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ESrJe7bq7y1x", + "outputId": "129c154a-d17f-4e60-c610-c3e5bb5bdb8f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Parsing and Text Indexing Files\n" + ] + }, + { + "data": { + "text/plain": [ + "{'docs_added': 0,\n", + " 'blocks_added': 0,\n", + " 'images_added': 0,\n", + " 'pages_added': 0,\n", + " 'tables_added': 0,\n", + " 'rejected_files': []}" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"update: Parsing and Text Indexing Files\")\n", + "library.add_files(input_folder_path=os.path.join(sample_files_path, \"Agreements\"), chunk_size=400, max_chunk_size=600, smart_chunking=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "id": "hBMo7HHv8TCy" + }, + "outputs": [], + "source": [ + "embedding_models = ModelCatalog().list_embedding_models()\n", + "embedding_model = \"mini-lm-sbert\"" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZDgZI1I68nBq", + "outputId": "88ccf03c-a4f4-46fc-992d-11747e1e22f0" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: Starting the Embedding: library - example2_library - vector_db - lancedb - model - mini-lm-sbert\n" + ] + } + ], + "source": [ + "library_name = library.library_name\n", + "vector_db = LLMWareConfig().get_vector_db()\n", + "print(f\"\\nupdate: Starting the Embedding: \"\n", + " f\"library - {library_name} - \"\n", + " f\"vector_db - {vector_db} - \"\n", + " f\"model - {embedding_model}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ySGrOKU4-6ig", + "outputId": "cb01d8fd-1f61-44e7-ddf3-77b166d739e4" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 100 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 200 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 300 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 400 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 500 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 600 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 700 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 800 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 900 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1000 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1100 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1200 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1300 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1400 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1500 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1600 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1700 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1800 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 1900 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 2000 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 2100 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 2200 of 2211\n", + "INFO:llmware.embeddings:update: embedding_handler - Lancedb - Embeddings Created: 2211 of 2211\n", + "INFO:llmware.embeddings:update: EmbeddingHandler - Lancedb - embedding_summary - {'embeddings_created': 2211, 'embedded_blocks': 2211, 'embedding_dims': 384, 'time_stamp': 'Thu Jun 6 16:09:59 2024'}\n" + ] + }, + { + "data": { + "text/plain": [ + "{'embeddings_created': 2211,\n", + " 'embedded_blocks': 2211,\n", + " 'embedding_dims': 384,\n", + " 'time_stamp': 'Thu Jun 6 16:09:59 2024'}" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db,batch_size=100)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lmppxssq--H3", + "outputId": "f825b091-ea64-48fe-8c64-3ed61a482c6b" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Embeddings Complete - Status() check at end of embedding - [{'_id': 2, 'key': 'example2_library_embedding_mini-lm-sbert', 'summary': '2211 of 2211 blocks', 'start_time': '1717690179.087806', 'end_time': '1717690199.5373614', 'total': 2211, 'current': 2211, 'units': 'blocks'}]\n" + ] + } + ], + "source": [ + "update = Status().get_embedding_status(library_name, embedding_model)\n", + "print(\"update: Embeddings Complete - Status() check at end of embedding - \", update)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "neWoHMwc_Mtm", + "outputId": "a7655116-4627-4c67-b173-de1b9a69bb4d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "update: Run a sample semantic/vectory query: incentive compensation\n" + ] + } + ], + "source": [ + "sample_query = \"incentive compensation\"\n", + "print(\"\\n\\nupdate: Run a sample semantic/vectory query: {}\".format(sample_query))" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "tPJEgjUb_WiI", + "outputId": "1f46d14c-596b-45ae-dae5-858429050589" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: query results - 0 - document - Artemis Poseidon EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.24837934970855713 \n", + "update: text sample - actual incentive bonus (“Incentive Bonus”) for any fiscal year as determined by the Board (or the compensation committee ... \n", + "\n", + "update: query results - 1 - document - Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.24837934970855713 \n", + "update: text sample - actual incentive bonus (“Incentive Bonus”) for any fiscal year as determined by the Board (or the compensation committee ... \n", + "\n", + "update: query results - 2 - document - Amphitrite EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.2483793944120407 \n", + "update: text sample - actual incentive bonus (“Incentive Bonus”) for any fiscal year as determined by the Board (or the compensation committee ... \n", + "\n", + "update: query results - 3 - document - Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.24960115551948547 \n", + "update: text sample - actual incentive bonus (“Incentive Bonus”) for any fiscal year as determined by the Board (or the compensation committee ... \n", + "\n", + "update: query results - 4 - document - Aphrodite EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.24960121512413025 \n", + "update: text sample - actual incentive bonus (“Incentive Bonus”) for any fiscal year as determined by the Board (or the compensation committee ... \n", + "\n", + "update: query results - 5 - document - Bia EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.2503372132778168 \n", + "update: text sample - actual incentive bonus (“Incentive Bonus”) for any fiscal year as determined by the Board (or the compensation committee ... \n", + "\n", + "update: query results - 6 - document - Persephone EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.269817054271698 \n", + "update: text sample - to participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in ex ... \n", + "\n", + "update: query results - 7 - document - Rhea EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.26981714367866516 \n", + "update: text sample - to participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in ex ... \n", + "\n", + "update: query results - 8 - document - Metis EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.26981714367866516 \n", + "update: text sample - to participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in ex ... \n", + "\n", + "update: query results - 9 - document - Gaia EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.27305811643600464 \n", + "update: text sample - in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in existence for oth ... \n", + "\n", + "update: query results - 10 - document - Eileithyia EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.27305811643600464 \n", + "update: text sample - in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in existence for oth ... \n", + "\n", + "update: query results - 11 - document - Demeter EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.2812928557395935 \n", + "update: text sample - participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in exist ... \n", + "\n", + "update: query results - 12 - document - Nike EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.28129294514656067 \n", + "update: text sample - participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in exist ... \n", + "\n", + "update: query results - 13 - document - Leto EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.28129294514656067 \n", + "update: text sample - participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in exist ... \n", + "\n", + "update: query results - 14 - document - Nyx EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 3 distance - 0.28129294514656067 \n", + "update: text sample - participate in Employer's annual cash incentive bonus plan (the “Plan”), based on the same terms and conditions as in exist ... \n", + "\n", + "update: query results - 15 - document - Gaia EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.3150967061519623 \n", + "update: text sample - target annual bonus under the Plan shall be 80% of Base Salary (“Target Bonus”), but Executive's actual incentive bonus (“I ... \n", + "\n", + "update: query results - 16 - document - Eileithyia EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.3150967061519623 \n", + "update: text sample - target annual bonus under the Plan shall be 80% of Base Salary (“Target Bonus”), but Executive's actual incentive bonus (“I ... \n", + "\n", + "update: query results - 17 - document - Metis EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.3150967061519623 \n", + "update: text sample - target annual bonus under the Plan shall be 80% of Base Salary (“Target Bonus”), but Executive's actual incentive bonus (“I ... \n", + "\n", + "update: query results - 18 - document - Rhea EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.3150967061519623 \n", + "update: text sample - target annual bonus under the Plan shall be 80% of Base Salary (“Target Bonus”), but Executive's actual incentive bonus (“I ... \n", + "\n", + "update: query results - 19 - document - Leto EXECUTIVE EMPLOYMENT AGREEMENT.pdf - page num - 4 distance - 0.31509676575660706 \n", + "update: text sample - target annual bonus under the Plan shall be 80% of Base Salary (“Target Bonus”), but Executive's actual incentive bonus (“I ... \n" + ] + } + ], + "source": [ + "query_results = Query(library).semantic_query(sample_query, result_count=20)\n", + "for i, entries in enumerate(query_results):\n", + " text = entries[\"text\"]\n", + " document_source = entries[\"file_source\"]\n", + " page_num = entries[\"page_num\"]\n", + " vector_distance = entries[\"distance\"]\n", + "\n", + " if len(text) > 125: text = text[0:125] + \" ... \"\n", + "\n", + " print(\"\\nupdate: query results - {} - document - {} - page num - {} distance - {} \".format(i, document_source, page_num, vector_distance))\n", + "\n", + " print(\"update: text sample - \", text)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "EpmAWSF0_8uP", + "outputId": "f80b9a23-e8c5-4020-eefe-a81579554967" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: embedding record - [{'embedding_status': 'yes', 'embedding_model': 'mini-lm-sbert', 'embedding_db': 'lancedb', 'embedding_dims': 384, 'embedded_blocks': 2211, 'time_stamp': 'Thu Jun 6 16:09:59 2024'}]\n" + ] + } + ], + "source": [ + "embedding_record = library.get_embedding_status()\n", + "print(\"\\nupdate: embedding record - \", embedding_record)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_2_build_embeddings_version_2.ipynb b/tutorials/notebooks/fast_start_examples/example_2_build_embeddings_version_2.ipynb new file mode 100644 index 0000000..50705ab --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_2_build_embeddings_version_2.ipynb @@ -0,0 +1,2915 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iwrhqAwE3DrS", + "outputId": "44eb96b3-9351-42c5-b71c-89f0e99045d7" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: llmware in /usr/local/lib/python3.10/dist-packages (0.3.0)\n", + "Requirement already satisfied: boto3>=1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.34.119)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.7.3)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.0)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.119 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.34.119)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (0.10.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.119->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.119->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.2.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.119->boto3>=1.24.53->llmware) (1.16.0)\n", + "Collecting chromadb\n", + " Downloading chromadb-0.5.0-py3-none-any.whl (526 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.8/526.8 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: build>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.2.1)\n", + "Requirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n", + "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.7.2)\n", + "Collecting chroma-hnswlib==0.7.3 (from chromadb)\n", + " Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m49.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n", + " Downloading fastapi-0.111.0-py3-none-any.whl (91 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n", + " Downloading uvicorn-0.30.1-py3-none-any.whl (62 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.4/62.4 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.22.5 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.25.2)\n", + "Collecting posthog>=2.4.0 (from chromadb)\n", + " Downloading posthog-3.5.0-py2.py3-none-any.whl (41 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.12.0)\n", + "Collecting onnxruntime>=1.14.1 (from chromadb)\n", + " Downloading onnxruntime-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (6.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m78.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb)\n", + " Downloading opentelemetry_api-1.25.0-py3-none-any.whl (59 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.9/59.9 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb)\n", + " Downloading opentelemetry_exporter_otlp_proto_grpc-1.25.0-py3-none-any.whl (18 kB)\n", + "Collecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb)\n", + " Downloading opentelemetry_instrumentation_fastapi-0.46b0-py3-none-any.whl (11 kB)\n", + "Collecting opentelemetry-sdk>=1.2.0 (from chromadb)\n", + " Downloading opentelemetry_sdk-1.25.0-py3-none-any.whl (107 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m107.0/107.0 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.19.1)\n", + "Collecting pypika>=0.48.9 (from chromadb)\n", + " Downloading PyPika-0.48.9.tar.gz (67 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.4)\n", + "Collecting overrides>=7.3.1 (from chromadb)\n", + " Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n", + "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.4.0)\n", + "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.64.0)\n", + "Collecting bcrypt>=4.0.1 (from chromadb)\n", + " Downloading bcrypt-4.1.3-cp39-abi3-manylinux_2_28_x86_64.whl (283 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m26.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.4)\n", + "Collecting kubernetes>=28.1.0 (from chromadb)\n", + " Downloading kubernetes-29.0.0-py2.py3-none-any.whl (1.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m71.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.3.0)\n", + "Requirement already satisfied: PyYAML>=6.0.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.0.1)\n", + "Collecting mmh3>=4.0.1 (from chromadb)\n", + " Downloading mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (67 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting orjson>=3.9.12 (from chromadb)\n", + " Downloading orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: packaging>=19.1 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (24.0)\n", + "Requirement already satisfied: pyproject_hooks in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (1.1.0)\n", + "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (2.0.1)\n", + "Collecting starlette<0.38.0,>=0.37.2 (from fastapi>=0.95.2->chromadb)\n", + " Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting fastapi-cli>=0.0.2 (from fastapi>=0.95.2->chromadb)\n", + " Downloading fastapi_cli-0.0.4-py3-none-any.whl (9.5 kB)\n", + "Collecting httpx>=0.23.0 (from fastapi>=0.95.2->chromadb)\n", + " Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: jinja2>=2.11.2 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.1.4)\n", + "Collecting python-multipart>=0.0.7 (from fastapi>=0.95.2->chromadb)\n", + " Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n", + "Collecting ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 (from fastapi>=0.95.2->chromadb)\n", + " Downloading ujson-5.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (53 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting email_validator>=2.0.0 (from fastapi>=0.95.2->chromadb)\n", + " Downloading email_validator-2.1.1-py3-none-any.whl (30 kB)\n", + "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2024.2.2)\n", + "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n", + "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.27.0)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n", + "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n", + "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n", + "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.0.7)\n", + "Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n", + " Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\n", + "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12.1)\n", + "Collecting deprecated>=1.2.6 (from opentelemetry-api>=1.2.0->chromadb)\n", + " Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n", + "Requirement already satisfied: importlib-metadata<=7.1,>=6.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-api>=1.2.0->chromadb) (7.1.0)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.63.0)\n", + "Collecting opentelemetry-exporter-otlp-proto-common==1.25.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n", + " Downloading opentelemetry_exporter_otlp_proto_common-1.25.0-py3-none-any.whl (17 kB)\n", + "Collecting opentelemetry-proto==1.25.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n", + " Downloading opentelemetry_proto-1.25.0-py3-none-any.whl (52 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-instrumentation-asgi==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_instrumentation_asgi-0.46b0-py3-none-any.whl (14 kB)\n", + "Collecting opentelemetry-instrumentation==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_instrumentation-0.46b0-py3-none-any.whl (29 kB)\n", + "Collecting opentelemetry-semantic-conventions==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_semantic_conventions-0.46b0-py3-none-any.whl (130 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.5/130.5 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-util-http==0.46b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_util_http-0.46b0-py3-none-any.whl (6.9 kB)\n", + "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (67.7.2)\n", + "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.14.1)\n", + "Collecting asgiref~=3.0 (from opentelemetry-instrumentation-asgi==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading asgiref-3.8.1-py3-none-any.whl (23 kB)\n", + "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n", + " Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n", + "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n", + " Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.18.3 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb) (2.18.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.7)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.23.2)\n", + "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n", + "Collecting h11>=0.8 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m36.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", + "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m76.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading watchfiles-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m68.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: dnspython>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from email_validator>=2.0.0->fastapi>=0.95.2->chromadb) (2.6.1)\n", + "Collecting typer>=0.9.0 (from chromadb)\n", + " Downloading typer-0.12.3-py3-none-any.whl (47 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting shellingham>=1.3.0 (from typer>=0.9.0->chromadb)\n", + " Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n", + "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (13.7.1)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.3)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n", + "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi>=0.95.2->chromadb) (3.7.1)\n", + "Collecting httpcore==1.* (from httpx>=0.23.0->fastapi>=0.95.2->chromadb)\n", + " Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi>=0.95.2->chromadb) (1.3.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=7.1,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.19.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.2->fastapi>=0.95.2->chromadb) (2.1.5)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (2.16.1)\n", + "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n", + " Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.23.0->fastapi>=0.95.2->chromadb) (1.2.1)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb) (0.1.2)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.0)\n", + "Building wheels for collected packages: pypika\n", + " Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53724 sha256=dd5c69c207bd4ac7eef6cec63c1953ba4b31c8f3a940b65c5a5dbc817a740b1b\n", + " Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n", + "Successfully built pypika\n", + "Installing collected packages: pypika, monotonic, mmh3, websockets, uvloop, ujson, shellingham, python-multipart, python-dotenv, overrides, orjson, opentelemetry-util-http, opentelemetry-proto, humanfriendly, httptools, h11, email_validator, deprecated, chroma-hnswlib, bcrypt, backoff, asgiref, watchfiles, uvicorn, starlette, posthog, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, httpcore, coloredlogs, typer, opentelemetry-semantic-conventions, opentelemetry-instrumentation, onnxruntime, kubernetes, httpx, opentelemetry-sdk, opentelemetry-instrumentation-asgi, fastapi-cli, opentelemetry-instrumentation-fastapi, opentelemetry-exporter-otlp-proto-grpc, fastapi, chromadb\n", + " Attempting uninstall: typer\n", + " Found existing installation: typer 0.9.4\n", + " Uninstalling typer-0.9.4:\n", + " Successfully uninstalled typer-0.9.4\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n", + "weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed asgiref-3.8.1 backoff-2.2.1 bcrypt-4.1.3 chroma-hnswlib-0.7.3 chromadb-0.5.0 coloredlogs-15.0.1 deprecated-1.2.14 email_validator-2.1.1 fastapi-0.111.0 fastapi-cli-0.0.4 h11-0.14.0 httpcore-1.0.5 httptools-0.6.1 httpx-0.27.0 humanfriendly-10.0 kubernetes-29.0.0 mmh3-4.1.0 monotonic-1.6 onnxruntime-1.18.0 opentelemetry-api-1.25.0 opentelemetry-exporter-otlp-proto-common-1.25.0 opentelemetry-exporter-otlp-proto-grpc-1.25.0 opentelemetry-instrumentation-0.46b0 opentelemetry-instrumentation-asgi-0.46b0 opentelemetry-instrumentation-fastapi-0.46b0 opentelemetry-proto-1.25.0 opentelemetry-sdk-1.25.0 opentelemetry-semantic-conventions-0.46b0 opentelemetry-util-http-0.46b0 orjson-3.10.3 overrides-7.7.0 posthog-3.5.0 pypika-0.48.9 python-dotenv-1.0.1 python-multipart-0.0.9 shellingham-1.5.4 starlette-0.37.2 typer-0.12.3 ujson-5.10.0 uvicorn-0.30.1 uvloop-0.19.0 watchfiles-0.22.0 websockets-12.0\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install chromadb" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "EDX0CEBe3MCS" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.library import Library\n", + "from llmware.retrieval import Query\n", + "from llmware.setup import Setup\n", + "from llmware.resources import Status\n", + "from llmware.models import ModelCatalog\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "lxtvCR_63M8P" + }, + "outputs": [], + "source": [ + "from importlib import util\n", + "if not util.find_spec(\"chromadb\"):\n", + " print(\"\\nto run this example with chromadb, you need to install the chromadb python sdk: pip3 install chromadb\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "ZiVBv75C30Af" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "LLMWareConfig().set_vector_db(\"chromadb\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "d74Eejvz4DLp", + "outputId": "e607d07e-84b4-4baf-c170-f72b7c08eb8a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: Creating library: example2_library\n" + ] + } + ], + "source": [ + "print(\"\\nupdate: Creating library: {}\".format(\"example2_library\"))\n", + "library = Library().create_new_library(\"example2_library\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "VD5GUdz-4i9k", + "outputId": "9a4a77b8-30ee-468f-f5eb-9472be2e9a23" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "embedding record - before embedding [{'embedding_status': 'no', 'embedding_model': 'none', 'embedding_db': 'none', 'embedded_blocks': 0, 'embedding_dims': 0, 'time_stamp': 'NA'}]\n" + ] + } + ], + "source": [ + "embedding_record = library.get_embedding_status()\n", + "print(\"embedding record - before embedding \", embedding_record)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ExNvXOrP4sSK", + "outputId": "11c79a56-816e-4413-a9c0-f74e84c72bf4" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Downloading Sample Files\n" + ] + } + ], + "source": [ + "print(\"update: Downloading Sample Files\")\n", + "sample_files_path = Setup().load_sample_files(over_write=False)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "yjPf3qb241wj", + "outputId": "f8ddcf95-414c-4d8e-af27-83ed734cde70" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Parsing and Text Indexing Files\n" + ] + }, + { + "data": { + "text/plain": [ + "{'docs_added': 15,\n", + " 'blocks_added': 2211,\n", + " 'images_added': 0,\n", + " 'pages_added': 204,\n", + " 'tables_added': 0,\n", + " 'rejected_files': []}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"update: Parsing and Text Indexing Files\")\n", + "library.add_files(input_folder_path=os.path.join(sample_files_path, \"Agreements\"), chunk_size=400, max_chunk_size=600, smart_chunking=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "id": "Mw-cCAH55MTp" + }, + "outputs": [], + "source": [ + "embedding_models = ModelCatalog().list_embedding_models()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "oXjhj7Yg5RAS" + }, + "outputs": [], + "source": [ + "embedding_model = \"mini-lm-sbert\"" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "id": "_xEVQoBf5Ww8" + }, + "outputs": [], + "source": [ + "library_name = library.library_name\n", + "vector_db = LLMWareConfig().get_vector_db()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 854, + "referenced_widgets": [ + "d64fa9bb007a485493a2e56d8060e4a4", + "a9bf4f1bdc0f4d5abcc2e2ecc094a3d0", + "abce106c862c41ceb4c9edd4738ce347", + "4b5ac9a2745d4f739066a0055358eea7", + "4373a021b2874a259756698587869ef6", + "40d7ae9643b04d12bbb2d8f352eedf7f", + "898a78046e904d1aaa868114c1cf792d", + "941dee4b8661492780557edef0002c67", + "aae91a396054443ebb15b20ceb8ba286", + "44220e6246514667b5f14d30897eb8ca", + "5fc792f757d34237a8ae56ab2ce2b96b", + "4d7c2830b10447b1b1717b03271a6edc", + "bd2300da8bd249b6ba920aecd4e111c0", + "da68a0dc157c4fb29cce9aea73b2b169", + "1fb7949521af45c186fe619e903cd3fe", + "fea1ae86322943559ce3e42586bd2091", + "024e9e990ff642d399496db123ba918c", + "039cabed48b843dc812614e852731616", + "2ab6c3418cb6415492b774b54e0d1686", + "950e402cecaf428492619e4393524776", + "9e7f0b3096a84863b8beebcb67832431", + "24bc1b00be084582b0989c64b83e0ad4", + "079745244401477e89ef7a08545a6e90", + "5620af8f53fe4f2daf9ab3496278350b", + "b25cda1faab444728bff6cb037e5fa63", + "9397ffbb10a548da8c91225370be25eb", + "6d1578c6acf44a38b56aa1f452860809", + "6481e8c8db844b568fe9a9d15db55406", + "abd061973be646cbb055c1519e17acd4", + "d2d56392f60e4032ab79cce5fc733bc1", + "4155ed216ec04d12aba48b47fea5b4b8", + "68829f6834ff453ab6a42c3fb564b679", + "fdb39c4b2db74b3d8c82b5eeadedc594", + "a9fef14c80df48c09a567c05c85d14be", + "9d47d80b1ff743c1a221be9d9b084cd2", + "26d8974ffc194c1580817ad2b3285047", + "03f91b2411d64c90bebce35db0cc2061", + "edd50d5af0d44d37925b112b491967f7", + "9a70a4c9481f4ff3b7de34b579288332", + "66495f39612b427288e2c654b64a81e7", + "32c5e4ea9f064822a97b2a45d36b019b", + "5d6cad4a0af04679b7f2acf2d4fe20d4", + "bf5d7518bb0e4f9ebd083530052b5bbd", + "711bc96801434ff79ec2f05bfb19ce69", + "44cb287e703c4711827e81b7ec194de5", + "051c207992d64ab6b14445b46eee8eb2", + "4dea19ac0d6448ccb5ff9c7c50190572", + "24fc5d103cee4981b6f7e1725512546a", + "37f5b9a3a83047cabc1277cba94ef677", + "f2fda07c4306441ca2286fe3e6cbd11d", + "f8d90501ed244fa49d6039e04693fd23", + "dba5580f26804029a9a64f1131a1c1af", + "dde5cf2ba9a547b4ad47f1c7e8d91b0c", + "185517d9c9f442cd86972b5a23e91115", + "81b4fa0427d148c287e6ef80330ddba6", + "c1ae0e7349b348d7ba8c16c6ae8824cb", + "19125679c9b049bf961ecb32ab6c6981", + "32f307d009824afc9506cab7f84f031a", + "d7995ba0a23b4bc0a7f9199ed1defc54", + "ecc7125b090648569da0a7ccbb4afd6c", + "45bcad94598b41dabef58ee74c0987c7", + "6cc55c1a92fe40bb8b421bc0c28c3eb7", + "76f6bde626fc41aca2339583f4d04558", + "b505d93248714dabad2a241ca0736700", + "e3666ea5568447519bb870145031c45f", + "fa6bd83c43b64c47a3263d7e90d14719" + ] + }, + "id": "_NfoTwjT5cQk", + "outputId": "3083b80b-052b-45a1-8f56-369042257f98" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: Starting the Embedding: library - example2_library -vector_db - chromadbmodel - mini-lm-sbert\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d64fa9bb007a485493a2e56d8060e4a4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "config.json: 0%| | 0.00/612 [00:00 125: text = text[0:125] + \" ... \"\n", + " print(\"\\nupdate: query results - {} - document - {} - page num - {} - distance - {} \".format( i, document_source, page_num, vector_distance))\n", + "\n", + " print(\"update: text sample - \", text)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "iZMfUkNY8GpY", + "outputId": "6044067b-0ada-41fd-a8c4-08cbbd5d2e0c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: embedding record - [{'embedding_status': 'yes', 'embedding_model': 'mini-lm-sbert', 'embedding_db': 'chromadb', 'embedding_dims': 384, 'embedded_blocks': 2211, 'time_stamp': 'Wed Jun 5 15:01:16 2024'}]\n" + ] + } + ], + "source": [ + "embedding_record = library.get_embedding_status()\n", + "print(\"\\nupdate: embedding record - \", embedding_record)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "024e9e990ff642d399496db123ba918c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "039cabed48b843dc812614e852731616": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "03f91b2411d64c90bebce35db0cc2061": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bf5d7518bb0e4f9ebd083530052b5bbd", + "placeholder": "​", + "style": "IPY_MODEL_711bc96801434ff79ec2f05bfb19ce69", + "value": " 232k/232k [00:00<00:00, 3.17MB/s]" + } + }, + "051c207992d64ab6b14445b46eee8eb2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f2fda07c4306441ca2286fe3e6cbd11d", + "placeholder": "​", + "style": "IPY_MODEL_f8d90501ed244fa49d6039e04693fd23", + "value": "tokenizer.json: 100%" + } + }, + "079745244401477e89ef7a08545a6e90": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5620af8f53fe4f2daf9ab3496278350b", + "IPY_MODEL_b25cda1faab444728bff6cb037e5fa63", + "IPY_MODEL_9397ffbb10a548da8c91225370be25eb" + ], + "layout": "IPY_MODEL_6d1578c6acf44a38b56aa1f452860809" + } + }, + "185517d9c9f442cd86972b5a23e91115": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "19125679c9b049bf961ecb32ab6c6981": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_45bcad94598b41dabef58ee74c0987c7", + "placeholder": "​", + "style": "IPY_MODEL_6cc55c1a92fe40bb8b421bc0c28c3eb7", + "value": "special_tokens_map.json: 100%" + } + }, + "1fb7949521af45c186fe619e903cd3fe": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9e7f0b3096a84863b8beebcb67832431", + "placeholder": "​", + "style": "IPY_MODEL_24bc1b00be084582b0989c64b83e0ad4", + "value": " 90.9M/90.9M [00:00<00:00, 117MB/s]" + } + }, + "24bc1b00be084582b0989c64b83e0ad4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "24fc5d103cee4981b6f7e1725512546a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_185517d9c9f442cd86972b5a23e91115", + "placeholder": "​", + "style": "IPY_MODEL_81b4fa0427d148c287e6ef80330ddba6", + "value": " 466k/466k [00:00<00:00, 5.19MB/s]" + } + }, + "26d8974ffc194c1580817ad2b3285047": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_32c5e4ea9f064822a97b2a45d36b019b", + "max": 231508, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5d6cad4a0af04679b7f2acf2d4fe20d4", + "value": 231508 + } + }, + "2ab6c3418cb6415492b774b54e0d1686": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "32c5e4ea9f064822a97b2a45d36b019b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "32f307d009824afc9506cab7f84f031a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_76f6bde626fc41aca2339583f4d04558", + "max": 112, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b505d93248714dabad2a241ca0736700", + "value": 112 + } + }, + "37f5b9a3a83047cabc1277cba94ef677": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "40d7ae9643b04d12bbb2d8f352eedf7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4155ed216ec04d12aba48b47fea5b4b8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4373a021b2874a259756698587869ef6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "44220e6246514667b5f14d30897eb8ca": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "44cb287e703c4711827e81b7ec194de5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_051c207992d64ab6b14445b46eee8eb2", + "IPY_MODEL_4dea19ac0d6448ccb5ff9c7c50190572", + "IPY_MODEL_24fc5d103cee4981b6f7e1725512546a" + ], + "layout": "IPY_MODEL_37f5b9a3a83047cabc1277cba94ef677" + } + }, + "45bcad94598b41dabef58ee74c0987c7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b5ac9a2745d4f739066a0055358eea7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_44220e6246514667b5f14d30897eb8ca", + "placeholder": "​", + "style": "IPY_MODEL_5fc792f757d34237a8ae56ab2ce2b96b", + "value": " 612/612 [00:00<00:00, 36.7kB/s]" + } + }, + "4d7c2830b10447b1b1717b03271a6edc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_bd2300da8bd249b6ba920aecd4e111c0", + "IPY_MODEL_da68a0dc157c4fb29cce9aea73b2b169", + "IPY_MODEL_1fb7949521af45c186fe619e903cd3fe" + ], + "layout": "IPY_MODEL_fea1ae86322943559ce3e42586bd2091" + } + }, + "4dea19ac0d6448ccb5ff9c7c50190572": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_dba5580f26804029a9a64f1131a1c1af", + "max": 466247, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_dde5cf2ba9a547b4ad47f1c7e8d91b0c", + "value": 466247 + } + }, + "5620af8f53fe4f2daf9ab3496278350b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6481e8c8db844b568fe9a9d15db55406", + "placeholder": "​", + "style": "IPY_MODEL_abd061973be646cbb055c1519e17acd4", + "value": "tokenizer_config.json: 100%" + } + }, + "5d6cad4a0af04679b7f2acf2d4fe20d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5fc792f757d34237a8ae56ab2ce2b96b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6481e8c8db844b568fe9a9d15db55406": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "66495f39612b427288e2c654b64a81e7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "68829f6834ff453ab6a42c3fb564b679": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6cc55c1a92fe40bb8b421bc0c28c3eb7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6d1578c6acf44a38b56aa1f452860809": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "711bc96801434ff79ec2f05bfb19ce69": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "76f6bde626fc41aca2339583f4d04558": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "81b4fa0427d148c287e6ef80330ddba6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "898a78046e904d1aaa868114c1cf792d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "9397ffbb10a548da8c91225370be25eb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_68829f6834ff453ab6a42c3fb564b679", + "placeholder": "​", + "style": "IPY_MODEL_fdb39c4b2db74b3d8c82b5eeadedc594", + "value": " 350/350 [00:00<00:00, 20.4kB/s]" + } + }, + "941dee4b8661492780557edef0002c67": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "950e402cecaf428492619e4393524776": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "9a70a4c9481f4ff3b7de34b579288332": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9d47d80b1ff743c1a221be9d9b084cd2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9a70a4c9481f4ff3b7de34b579288332", + "placeholder": "​", + "style": "IPY_MODEL_66495f39612b427288e2c654b64a81e7", + "value": "vocab.txt: 100%" + } + }, + "9e7f0b3096a84863b8beebcb67832431": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a9bf4f1bdc0f4d5abcc2e2ecc094a3d0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_40d7ae9643b04d12bbb2d8f352eedf7f", + "placeholder": "​", + "style": "IPY_MODEL_898a78046e904d1aaa868114c1cf792d", + "value": "config.json: 100%" + } + }, + "a9fef14c80df48c09a567c05c85d14be": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_9d47d80b1ff743c1a221be9d9b084cd2", + "IPY_MODEL_26d8974ffc194c1580817ad2b3285047", + "IPY_MODEL_03f91b2411d64c90bebce35db0cc2061" + ], + "layout": "IPY_MODEL_edd50d5af0d44d37925b112b491967f7" + } + }, + "aae91a396054443ebb15b20ceb8ba286": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "abce106c862c41ceb4c9edd4738ce347": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_941dee4b8661492780557edef0002c67", + "max": 612, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_aae91a396054443ebb15b20ceb8ba286", + "value": 612 + } + }, + "abd061973be646cbb055c1519e17acd4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b25cda1faab444728bff6cb037e5fa63": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d2d56392f60e4032ab79cce5fc733bc1", + "max": 350, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4155ed216ec04d12aba48b47fea5b4b8", + "value": 350 + } + }, + "b505d93248714dabad2a241ca0736700": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "bd2300da8bd249b6ba920aecd4e111c0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_024e9e990ff642d399496db123ba918c", + "placeholder": "​", + "style": "IPY_MODEL_039cabed48b843dc812614e852731616", + "value": "model.safetensors: 100%" + } + }, + "bf5d7518bb0e4f9ebd083530052b5bbd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c1ae0e7349b348d7ba8c16c6ae8824cb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_19125679c9b049bf961ecb32ab6c6981", + "IPY_MODEL_32f307d009824afc9506cab7f84f031a", + "IPY_MODEL_d7995ba0a23b4bc0a7f9199ed1defc54" + ], + "layout": "IPY_MODEL_ecc7125b090648569da0a7ccbb4afd6c" + } + }, + "d2d56392f60e4032ab79cce5fc733bc1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d64fa9bb007a485493a2e56d8060e4a4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a9bf4f1bdc0f4d5abcc2e2ecc094a3d0", + "IPY_MODEL_abce106c862c41ceb4c9edd4738ce347", + "IPY_MODEL_4b5ac9a2745d4f739066a0055358eea7" + ], + "layout": "IPY_MODEL_4373a021b2874a259756698587869ef6" + } + }, + "d7995ba0a23b4bc0a7f9199ed1defc54": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e3666ea5568447519bb870145031c45f", + "placeholder": "​", + "style": "IPY_MODEL_fa6bd83c43b64c47a3263d7e90d14719", + "value": " 112/112 [00:00<00:00, 6.96kB/s]" + } + }, + "da68a0dc157c4fb29cce9aea73b2b169": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2ab6c3418cb6415492b774b54e0d1686", + "max": 90868376, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_950e402cecaf428492619e4393524776", + "value": 90868376 + } + }, + "dba5580f26804029a9a64f1131a1c1af": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "dde5cf2ba9a547b4ad47f1c7e8d91b0c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e3666ea5568447519bb870145031c45f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ecc7125b090648569da0a7ccbb4afd6c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "edd50d5af0d44d37925b112b491967f7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f2fda07c4306441ca2286fe3e6cbd11d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f8d90501ed244fa49d6039e04693fd23": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fa6bd83c43b64c47a3263d7e90d14719": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fdb39c4b2db74b3d8c82b5eeadedc594": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fea1ae86322943559ce3e42586bd2091": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_3_prompts_and_models_version_1.ipynb b/tutorials/notebooks/fast_start_examples/example_3_prompts_and_models_version_1.ipynb new file mode 100644 index 0000000..605003a --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_3_prompts_and_models_version_1.ipynb @@ -0,0 +1,2744 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "wZ58h5LMwBhx" + }, + "source": [ + "# **If you are using Colab for free, we highly recommend you active the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed to the 13GB Colab gives automatically.**\n", + "# **To active T4:**\n", + "# **1. click on the \"Runtime\" tab**\n", + "# **2. click on \"Change runtime type\"**\n", + "# **3. select T4 GPU under Hardware Accelerator**\n", + "# **NOTE: there is a weekly usage limit on using T4 for free**\n", + "\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WIyD5iacAKHg", + "outputId": "7fd9b314-8b68-4e3d-e29f-3edad144d316" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.3.0-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.120-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.3/139.3 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (669 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m669.1/669.1 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m53.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "Collecting botocore<1.35.0,>=1.34.120 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.120-py3-none-any.whl (12.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.3/12.3 MB\u001b[0m \u001b[31m44.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.1-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.2/82.2 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m33.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.120->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.120 botocore-1.34.120 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.0 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.3 s3transfer-0.10.1\n", + "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.3.0+cu121)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.14.0)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.12.1)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12.1)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.4)\n", + "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch)\n", + " Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch)\n", + " Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch)\n", + " Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch)\n", + " Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch)\n", + " Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch)\n", + " Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "Collecting nvidia-curand-cu12==10.3.2.106 (from torch)\n", + " Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch)\n", + " Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch)\n", + " Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "Collecting nvidia-nccl-cu12==2.20.5 (from torch)\n", + " Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n", + "Collecting nvidia-nvtx-cu12==12.1.105 (from torch)\n", + " Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.3.0)\n", + "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch)\n", + " Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m55.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n", + "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n", + "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12\n", + "Successfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105\n", + "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.41.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.14.0)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2024.5.15)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n", + "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n", + "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (2023.6.0)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.23.0->transformers) (4.12.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.6.2)\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install torch\n", + "!pip install transformers" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OjhR95nTB5Ye" + }, + "outputs": [], + "source": [ + "import time\n", + "from llmware.prompts import Prompt\n", + "from llmware.models import ModelCatalog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Vu5OlNLsB_rL" + }, + "outputs": [], + "source": [ + "llm_models = ModelCatalog().list_generative_models()\n", + "llm_local_models = ModelCatalog().list_generative_local_models()\n", + "llm_open_source_models = ModelCatalog().list_open_source_models()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "GX--TIaACSvr", + "outputId": "a9fa3e82-93fd-4852-a182-3ea6d3f9a852" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "models: 0 Meta-Llama-3-8B HFGenerativeModel\n", + "models: 1 Meta-Llama-3-8B-Instruct HFGenerativeModel\n", + "models: 2 QuantFactory/Meta-Llama-3-8B-GGUF GGUFGenerativeModel\n", + "models: 3 QuantFactory/Meta-Llama-3-8B-Instruct-GGUF GGUFGenerativeModel\n", + "models: 4 TheBloke/Llama-2-7B-Chat-GGUF GGUFGenerativeModel\n", + "models: 5 TheBloke/OpenHermes-2.5-Mistral-7B-GGUF GGUFGenerativeModel\n", + "models: 6 TheBloke/Starling-LM-7B-alpha-GGUF GGUFGenerativeModel\n", + "models: 7 TheBloke/zephyr-7B-beta-GGUF GGUFGenerativeModel\n", + "models: 8 bartowski/Meta-Llama-3-8B-Instruct-GGUF GGUFGenerativeModel\n", + "models: 9 bling-answer-tool GGUFGenerativeModel\n", + "models: 10 bling-phi-3-gguf GGUFGenerativeModel\n", + "models: 11 bling-stablelm-3b-tool GGUFGenerativeModel\n", + "models: 12 dragon-llama-answer-tool GGUFGenerativeModel\n", + "models: 13 dragon-mistral-answer-tool GGUFGenerativeModel\n", + "models: 14 dragon-yi-answer-tool GGUFGenerativeModel\n", + "models: 15 llmware/bling-1.4b-0.1 HFGenerativeModel\n", + "models: 16 llmware/bling-1b-0.1 HFGenerativeModel\n", + "models: 17 llmware/bling-cerebras-1.3b-0.1 HFGenerativeModel\n", + "models: 18 llmware/bling-falcon-1b-0.1 HFGenerativeModel\n", + "models: 19 llmware/bling-phi-3 HFGenerativeModel\n", + "models: 20 llmware/bling-red-pajamas-3b-0.1 HFGenerativeModel\n", + "models: 21 llmware/bling-sheared-llama-1.3b-0.1 HFGenerativeModel\n", + "models: 22 llmware/bling-sheared-llama-2.7b-0.1 HFGenerativeModel\n", + "models: 23 llmware/bling-stable-lm-3b-4e1t-v0 HFGenerativeModel\n", + "models: 24 llmware/bling-tiny-llama-v0 HFGenerativeModel\n", + "models: 25 llmware/dragon-deci-6b-v0 HFGenerativeModel\n", + "models: 26 llmware/dragon-deci-7b-v0 HFGenerativeModel\n", + "models: 27 llmware/dragon-falcon-7b-v0 HFGenerativeModel\n", + "models: 28 llmware/dragon-llama-7b-gguf GGUFGenerativeModel\n", + "models: 29 llmware/dragon-llama-7b-v0 HFGenerativeModel\n", + "models: 30 llmware/dragon-mistral-7b-gguf GGUFGenerativeModel\n", + "models: 31 llmware/dragon-mistral-7b-v0 HFGenerativeModel\n", + "models: 32 llmware/dragon-red-pajama-7b-v0 HFGenerativeModel\n", + "models: 33 llmware/dragon-stablelm-7b-v0 HFGenerativeModel\n", + "models: 34 llmware/dragon-yi-6b-gguf GGUFGenerativeModel\n", + "models: 35 llmware/dragon-yi-6b-v0 HFGenerativeModel\n", + "models: 36 llmware/slim-category HFGenerativeModel\n", + "models: 37 llmware/slim-emotions HFGenerativeModel\n", + "models: 38 llmware/slim-intent HFGenerativeModel\n", + "models: 39 llmware/slim-ner HFGenerativeModel\n", + "models: 40 llmware/slim-nli HFGenerativeModel\n", + "models: 41 llmware/slim-q-gen-phi-3 HFGenerativeModel\n", + "models: 42 llmware/slim-q-gen-tiny HFGenerativeModel\n", + "models: 43 llmware/slim-qa-gen-phi-3 HFGenerativeModel\n", + "models: 44 llmware/slim-qa-gen-tiny HFGenerativeModel\n", + "models: 45 llmware/slim-ratings HFGenerativeModel\n", + "models: 46 llmware/slim-sentiment HFGenerativeModel\n", + "models: 47 llmware/slim-sql-1b-v0 HFGenerativeModel\n", + "models: 48 llmware/slim-tags HFGenerativeModel\n", + "models: 49 llmware/slim-topics HFGenerativeModel\n", + "models: 50 microsoft/Phi-3-mini-128k-instruct HFGenerativeModel\n", + "models: 51 microsoft/Phi-3-mini-4k-instruct HFGenerativeModel\n", + "models: 52 microsoft/Phi-3-mini-4k-instruct-gguf GGUFGenerativeModel\n", + "models: 53 slim-boolean HFGenerativeModel\n", + "models: 54 slim-boolean-tool GGUFGenerativeModel\n", + "models: 55 slim-category-tool GGUFGenerativeModel\n", + "models: 56 slim-emotions-tool GGUFGenerativeModel\n", + "models: 57 slim-extract HFGenerativeModel\n", + "models: 58 slim-extract-tool GGUFGenerativeModel\n", + "models: 59 slim-intent-tool GGUFGenerativeModel\n", + "models: 60 slim-ner-tool GGUFGenerativeModel\n", + "models: 61 slim-nli-tool GGUFGenerativeModel\n", + "models: 62 slim-q-gen-phi-3-tool GGUFGenerativeModel\n", + "models: 63 slim-q-gen-tiny-tool GGUFGenerativeModel\n", + "models: 64 slim-qa-gen-phi-3-tool GGUFGenerativeModel\n", + "models: 65 slim-qa-gen-tiny-tool GGUFGenerativeModel\n", + "models: 66 slim-ratings-tool GGUFGenerativeModel\n", + "models: 67 slim-sa-ner HFGenerativeModel\n", + "models: 68 slim-sa-ner-tool GGUFGenerativeModel\n", + "models: 69 slim-sentiment-tool GGUFGenerativeModel\n", + "models: 70 slim-sql-tool GGUFGenerativeModel\n", + "models: 71 slim-summary HFGenerativeModel\n", + "models: 72 slim-summary-tool GGUFGenerativeModel\n", + "models: 73 slim-tags-3b HFGenerativeModel\n", + "models: 74 slim-tags-3b-tool GGUFGenerativeModel\n", + "models: 75 slim-tags-tool GGUFGenerativeModel\n", + "models: 76 slim-topics-tool GGUFGenerativeModel\n", + "models: 77 slim-xsum HFGenerativeModel\n", + "models: 78 slim-xsum-tool GGUFGenerativeModel\n", + "models: 79 tiny-llama-chat-gguf GGUFGenerativeModel\n", + "models: 80 whisper-cpp-base WhisperCPPModel\n", + "models: 81 whisper-cpp-base-english WhisperCPPModel\n", + "models: 82 whisper-cpp-tiny-diarize WhisperCPPModel\n" + ] + } + ], + "source": [ + "for i, models in enumerate(llm_local_models):\n", + " print(\"models: \", i, models[\"model_name\"], models[\"model_family\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "l0Rv8Cd6CoqT" + }, + "outputs": [], + "source": [ + "pytorch_generative_models = [\"llmware/bling-1b-0.1\", \"llmware/bling-tiny-llama-v0\", \"llmware/bling-falcon-1b-0.1\"]\n", + "gguf_generative_models = [\"bling-answer-tool\", \"bling-phi-3-gguf\",\"llmware/dragon-yi-6b-gguf\"]\n", + "model_name = gguf_generative_models[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_VrUvFMrC4Lq" + }, + "outputs": [], + "source": [ + "t0 = time.time()\n", + "test_list = [\n", + "\n", + "{\"query\": \"What is the total amount of the invoice?\",\n", + " \"answer\": \"$22,500.00\",\n", + " \"context\": \"Services Vendor Inc. \\n100 Elm Street Pleasantville, NY \\nTO Alpha Inc. 5900 1st Street \"\n", + " \"Los Angeles, CA \\nDescription Front End Engineering Service $5000.00 \\n Back End Engineering\"\n", + " \" Service $7500.00 \\n Quality Assurance Manager $10,000.00 \\n Total Amount $22,500.00 \\n\"\n", + " \"Make all checks payable to Services Vendor Inc. Payment is due within 30 days.\"\n", + " \"If you have any questions concerning this invoice, contact Bia Hermes. \"\n", + " \"THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000\"},\n", + "\n", + "{\"query\": \"What was the amount of the trade surplus?\",\n", + " \"answer\": \"62.4 billion yen ($416.6 million)\",\n", + " \"context\": \"Japan’s September trade balance swings into surplus, surprising expectations\"\n", + " \"Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, \"\n", + " \"beating expectations from economists polled by Reuters for a trade deficit of 42.5 \"\n", + " \"billion yen. Data from Japan’s customs agency revealed that exports in September \"\n", + " \"increased 4.3% year on year, while imports slid 16.3% compared to the same period \"\n", + " \"last year. According to FactSet, exports to Asia fell for the ninth straight month, \"\n", + " \"which reflected ongoing China weakness. Exports were supported by shipments to \"\n", + " \"Western markets, FactSet added. — Lim Hui Jie\"},\n", + "\n", + "{\"query\": \"What was Microsoft's revenue in the 3rd quarter?\",\n", + " \"answer\": \"$52.9 billion\",\n", + " \"context\": \"Microsoft Cloud Strength Drives Third Quarter Results \\nREDMOND, Wash. — April 25, 2023 — \"\n", + " \"Microsoft Corp. today announced the following results for the quarter ended March 31, 2023,\"\n", + " \" as compared to the corresponding period of last fiscal year:\\n· Revenue was $52.9 billion\"\n", + " \" and increased 7% (up 10% in constant currency)\\n· Operating income was $22.4 billion \"\n", + " \"and increased 10% (up 15% in constant currency)\\n· Net income was $18.3 billion and \"\n", + " \"increased 9% (up 14% in constant currency)\\n· Diluted earnings per share was $2.45 \"\n", + " \"and increased 10% (up 14% in constant currency).\\n\"},\n", + "\n", + "{\"query\": \"When did the LISP machine market collapse?\",\n", + " \"answer\": \"1987.\",\n", + " \"context\": \"The attendees became the leaders of AI research in the 1960s.\"\n", + " \" They and their students produced programs that the press described as 'astonishing': \"\n", + " \"computers were learning checkers strategies, solving word problems in algebra, \"\n", + " \"proving logical theorems and speaking English. By the middle of the 1960s, research in \"\n", + " \"the U.S. was heavily funded by the Department of Defense and laboratories had been \"\n", + " \"established around the world. Herbert Simon predicted, 'machines will be capable, \"\n", + " \"within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, \"\n", + " \"'within a generation ... the problem of creating 'artificial intelligence' will \"\n", + " \"substantially be solved'. They had, however, underestimated the difficulty of the problem. \"\n", + " \"Both the U.S. and British governments cut off exploratory research in response \"\n", + " \"to the criticism of Sir James Lighthill and ongoing pressure from the US Congress \"\n", + " \"to fund more productive projects. Minsky's and Papert's book Perceptrons was understood \"\n", + " \"as proving that artificial neural networks approach would never be useful for solving \"\n", + " \"real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period \"\n", + " \"when obtaining funding for AI projects was difficult, followed. In the early 1980s, \"\n", + " \"AI research was revived by the commercial success of expert systems, a form of AI \"\n", + " \"program that simulated the knowledge and analytical skills of human experts. By 1985, \"\n", + " \"the market for AI had reached over a billion dollars. At the same time, Japan's fifth \"\n", + " \"generation computer project inspired the U.S. and British governments to restore funding \"\n", + " \"for academic research. However, beginning with the collapse of the Lisp Machine market \"\n", + " \"in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began.\"},\n", + "\n", + "{\"query\": \"When will employment start?\",\n", + " \"answer\": \"April 16, 2012.\",\n", + " \"context\": \"THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered \"\n", + " \"into this 2nd day of April, 2012, by and between Aphrodite Apollo \"\n", + " \"(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), \"\n", + " \"and shall become effective upon Executive’s commencement of employment \"\n", + " \"(the “Effective Date”) which is expected to commence on April 16, 2012. \"\n", + " \"The Company and Executive agree that unless Executive has commenced \"\n", + " \"employment with the Company as of April 16, 2012 (or such later date as \"\n", + " \"agreed by each of the Company and Executive) this Agreement shall be \"\n", + " \"null and void and of no further effect.\"},\n", + "\n", + "{\"query\": \"What is the current rate on 10-year treasuries?\",\n", + " \"answer\": \"4.58%\",\n", + " \"context\": \"Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data \"\n", + " \"and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, \"\n", + " \"or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy \"\n", + " \"Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in \"\n", + " \"August, the Labor Department said. Economists polled by Dow Jones expected 273,000 \"\n", + " \"jobs. However, wages rose less than expected last month. Stocks posted a stunning \"\n", + " \"turnaround on Friday, after initially falling on the stronger-than-expected jobs report. \"\n", + " \"At its session low, the Dow had fallen as much as 198 points; it surged by more than \"\n", + " \"500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during \"\n", + " \"their lowest points in the day. Traders were unclear of the reason for the intraday \"\n", + " \"reversal. Some noted it could be the softer wage number in the jobs report that made \"\n", + " \"investors rethink their earlier bearish stance. Others noted the pullback in yields from \"\n", + " \"the day’s highs. Part of the rally may just be to do a market that had gotten extremely \"\n", + " \"oversold with the S&P 500 at one point this week down more than 9% from its high earlier \"\n", + " \"this year. Yields initially surged after the report, with the 10-year Treasury rate trading \"\n", + " \"near its highest level in 14 years. The benchmark rate later eased from those levels, but \"\n", + " \"was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back \"\n", + " \"in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s \"\n", + " \"helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries \"\n", + " \"Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially \"\n", + " \"some oversold conditions.'\"},\n", + "\n", + "{\"query\": \"What is the governing law?\",\n", + " \"answer\": \"State of Massachusetts\",\n", + " \"context\": \"19.\tGoverning Law and Procedures. This Agreement shall be governed by and interpreted \"\n", + " \"under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement,\"\n", + " \" which shall be governed by the laws of the State of Delaware, without giving effect to any \"\n", + " \"conflict of laws provisions. Employer and Executive each irrevocably and unconditionally \"\n", + " \"(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief \"\n", + " \"or other equitable relief related to this Agreement or any action commenced by Executive pursuant \"\n", + " \"to any provision hereof, may be brought in the United States District Court for the federal \"\n", + " \"district in which Executive’s principal place of employment is located, or if such court does \"\n", + " \"not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction \"\n", + " \"in the state and county in which Executive’s principal place of employment is located, \"\n", + " \"(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o\"\n", + " \"r proceeding, and (c) waives any objection which Employer or Executive may have to the \"\n", + " \"laying of venue of any such suit, action or proceeding in any such court. Employer and \"\n", + " \"Executive each also irrevocably and unconditionally consents to the service of any process, \"\n", + " \"pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8.\"},\n", + "\n", + "{\"query\": \"What is the amount of the base salary?\",\n", + " \"answer\": \"$200,000.\",\n", + " \"context\": \"2.2. Base Salary. For all the services rendered by Executive hereunder, during the \"\n", + " \"Employment Period, Employer shall pay Executive a base salary at the annual rate of \"\n", + " \"$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. \"\n", + " \"Executive’s base salary shall be reviewed annually by the Board (or the compensation committee \"\n", + " \"of the Board), pursuant to Employer’s normal compensation and performance review policies \"\n", + " \"for senior level executives, and may be increased but not decreased. The amount of any \"\n", + " \"increase for each year shall be determined accordingly. For purposes of this Agreement, \"\n", + " \"the term “Base Salary” shall mean the amount of Executive’s base salary established \"\n", + " \"from time to time pursuant to this Section 2.2. \"},\n", + "\n", + "{\"query\": \"Is the expected gross margin greater than 70%?\",\n", + " \"answer\": \"Yes, between 71.5% and 72.%\",\n", + " \"context\": \"Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:\"\n", + " \"Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP \"\n", + " \"gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus \"\n", + " \"50 basis points. GAAP and non-GAAP operating expenses are expected to be \"\n", + " \"approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP \"\n", + " \"other income and expense are expected to be an income of approximately $100 \"\n", + " \"million, excluding gains and losses from non-affiliated investments. GAAP and \"\n", + " \"non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items.\"\n", + " \"Highlights NVIDIA achieved progress since its previous earnings announcement \"\n", + " \"in these areas: Data Center Second-quarter revenue was a record $10.32 billion, \"\n", + " \"up 141% from the previous quarter and up 171% from a year ago. Announced that the \"\n", + " \"NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping \"\n", + " \"this quarter, with a second-generation version with HBM3e memory expected to ship \"\n", + " \"in Q2 of calendar 2024. \"},\n", + "\n", + "{\"query\": \"What is Bank of America's rating on Target?\",\n", + " \"answer\": \"Buy\",\n", + " \"context\": \"Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from \"\n", + " \"my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom \"\n", + " \"of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index \"\n", + " \"soared more than 22%. Hotter than expected September consumer price index, consumer \"\n", + " \"inflation. The Social Security Administration issues announced a 3.2% cost-of-living \"\n", + " \"adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. \"\n", + " \"Cites consumer price index showing sticky retail inflation for the fourth time \"\n", + " \"in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites \"\n", + " \"risk/reward from depressed levels. Traffic could improve. Gross margin upside. \"\n", + " \"Merchandising better. Freight and transportation better. Target to report quarter \"\n", + " \"next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), \"\n", + " \"the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs \"\n", + " \"tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, \"\n", + " \"Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating.\"\n", + " \"If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the \"\n", + " \"Market email newsletter for free. Barclays cuts price targets on consumer products: \"\n", + " \"UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from \"\n", + " \"$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. \"\n", + " \"Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers\"\n", + " \"(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek\"\n", + " \"(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on \"\n", + " \"third quarter of 19-cent per share drag on earnings. The buyer: investors led by \"\n", + " \"private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for \"\n", + " \"Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share \"\n", + " \"from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps \"\n", + " \"overweight (buy) rating but lowers price target to $139 per share from $150. \"\n", + " \"Sees “still challenging” environment into third-quarter print. The Club owns shares \"\n", + " \"in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) \"\n", + " \"to overweight from equal weight (buy from hold) but lowers price target to $224 per \"\n", + " \"share from $230. Risk reward upgrade. Best visibility of utility scale names.\"},\n", + "\n", + "{\"query\": \"Who is NVIDIA's partner for the driver assistance system?\",\n", + " \"answer\": \"MediaTek\",\n", + " \"context\": \"Automotive Second-quarter revenue was $253 million, down 15% from the previous \"\n", + " \"quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering \"\n", + " \"the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. \"\n", + " \"Partnered with MediaTek, which will develop mainstream automotive systems on \"\n", + " \"chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics.\"},\n", + "\n", + "{\"query\": \"What was the rate of decline in 3rd quarter sales?\",\n", + " \"answer\": \"20% year-on-year.\",\n", + " \"context\": \"Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following \"\n", + " \"third quarter earnings that plunged. The Finnish telecommunications giant said that \"\n", + " \"it will reduce its cost base and increase operation efficiency to “address the \"\n", + " \"challenging market environment. The substantial layoffs come after Nokia reported \"\n", + " \"third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over \"\n", + " \"the period plunged by 69% year-on-year to 133 million euros.\"},\n", + "\n", + "{\"query\": \"What was professional visualization revenue in the quarter?\",\n", + " \"answer\": \"$379 million\",\n", + " \"context\": \"Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up \"\n", + " \"22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, \"\n", + " \"bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299.\"\n", + " \"Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model \"\n", + " \"foundry service using AI-powered natural language interactions to transform games \"\n", + " \"by bringing intelligence to non-playable characters. Added 35 DLSS games, including \"\n", + " \"Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: \"\n", + " \"Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool.\"\n", + " \"Professional Visualization Second-quarter revenue was $379 million, up 28% from the \"\n", + " \"previous quarter and down 24% from a year ago. Announced three new desktop \"\n", + " \"workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 \"\n", + " \"and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are \"\n", + " \"shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, \"\n", + " \"with new foundation applications and services for developers and industrial \"\n", + " \"enterprises to optimize and enhance their 3D pipelines with OpenUSD and \"\n", + " \"generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the \"\n", + " \"Alliance for OpenUSD to promote the standardization, development, evolution and \"\n", + " \"growth of Universal Scene Description technology.\"},\n", + "\n", + "\n", + "{\"query\": \"What is the executive's title?\",\n", + " \"answer\": \"Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.\",\n", + " \"context\": \"2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, \"\n", + " \"Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s \"\n", + " \"Workforce Optimization Division. In such role, Executive will report to the Board of \"\n", + " \"Directors of Employer (the “Board”) and shall devote substantially all of his business time \"\n", + " \"and attention and his best efforts and ability to the operations of Employer and its subsidiaries. \"\n", + " \"Executive shall be responsible for running Employer’s day-to-day operations and shall perform \"\n", + " \"faithfully, diligently and competently the duties and responsibilities of a SVP and such other \"\n", + " \"duties and responsibilities as directed by the Board and are consistent with such position. \"\n", + " \"The foregoing shall not be construed as preventing Executive from (a) making passive \"\n", + " \"investments in other businesses or enterprises consistent with Employer’s code of conduct, \"\n", + " \"or (b) engaging in any other business activity consistent with Employer’s code of conduct; \"\n", + " \"provided that Executive seeks and obtains the prior approval of the Board before engaging \"\n", + " \"in any other business activity. In addition, it shall not be a violation of this Agreement \"\n", + " \"for Executive to participate in civic or charitable activities, deliver lectures, fulfill \"\n", + " \"speaking engagements, teach at educational institutions, and/or manage personal investments \"\n", + " \"(subject to the immediately preceding sentence); provided that such activities do not \"\n", + " \"interfere in any substantial respect with the performance of Executive’s responsibilities \"\n", + " \"as an employee in accordance with this Agreement. Executive may also serve on one or more \"\n", + " \"corporate boards of another company (and committees thereof) upon giving advance notice \"\n", + " \"to the Board prior to commencing service on any other corporate board.\"},\n", + "\n", + "{\"query\": \"According to the CFO, what led to the increase in cloud revenue?\",\n", + " \"answer\": \"Focused execution by our sales teams and partners\",\n", + " \"context\": \"'The world's most advanced AI models \"\n", + " \"are coming together with the world's most universal user interface - natural language - \"\n", + " \"to create a new era of computing,' said Satya Nadella, chairman and chief \"\n", + " \"executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform \"\n", + " \"of choice to help customers get the most value out of their digital spend and innovate \"\n", + " \"for this next generation of AI.' 'Focused execution by our sales teams and partners \"\n", + " \"in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, \"\n", + " \"up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive \"\n", + " \"vice president and chief financial officer of Microsoft.\\n\"},\n", + "\n", + "{\"query\": \"Which company is located in Nevada?\",\n", + " \"answer\": \"North Industries\",\n", + " \"context\": \"To send notices to Blue Moon Tech, mail to their headquarters at: \"\n", + " \"555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to\"\n", + " \"their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\\nTo send notices \"\n", + " \"to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234.\"},\n", + "\n", + "{\"query\": \"When can termination after a material breach occur?\",\n", + " \"answer\": \"If the breach is not cured within 15 days of notice of the breach.\",\n", + " \"context\": \"This Agreement shall remain in effect until terminated. Either party may terminate this \"\n", + " \"agreement, any Statement of Work or Services Description for convenience by giving the other \"\n", + " \"party 30 days written notice. Either party may terminate this Agreement or any work order or \"\n", + " \"services description if the other party is in material breach or default of any obligation \"\n", + " \"that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees \"\n", + " \"for services performed and expenses incurred prior to the termination of this Agreement. \"\n", + " \"Termination of this Agreement will terminate all outstanding Statement of Work or Services \"\n", + " \"Description entered into under this agreement.\"},\n", + "\n", + "{\"query\": \"What is a headline summary in 10 words or less?\",\n", + " \"answer\": \"Joe Biden is the 46th President of the United States.\",\n", + " \"context\": \"Joe Biden's tenure as the 46th president of the United States began with \"\n", + " \"his inauguration on January 20, 2021. Biden, a Democrat from Delaware who \"\n", + " \"previously served as vice president under Barack Obama, \"\n", + " \"took office following his victory in the 2020 presidential election over \"\n", + " \"Republican incumbent president Donald Trump. Upon his inauguration, he \"\n", + " \"became the oldest president in American history.\"},\n", + "\n", + "{\"query\": \"Who are the two people that won elections in Georgia?\",\n", + " \"answer\": \"Jon Ossoff and Raphael Warnock\",\n", + " \"context\": \"Though Biden was generally acknowledged as the winner, \"\n", + " \"General Services Administration head Emily W. Murphy \"\n", + " \"initially refused to begin the transition to the president-elect, \"\n", + " \"thereby denying funds and office space to his team. \"\n", + " \"On November 23, after Michigan certified its results, Murphy \"\n", + " \"issued the letter of ascertainment, granting the Biden transition \"\n", + " \"team access to federal funds and resources for an orderly transition. \"\n", + " \"Two days after becoming the projected winner of the 2020 election, \"\n", + " \"Biden announced the formation of a task force to advise him on the \"\n", + " \"COVID-19 pandemic during the transition, co-chaired by former \"\n", + " \"Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, \"\n", + " \"and Yale University's Marcella Nunez-Smith. On January 5, 2021, \"\n", + " \"the Democratic Party won control of the United States Senate, \"\n", + " \"effective January 20, as a result of electoral victories in \"\n", + " \"Georgia by Jon Ossoff in a runoff election for a six-year term \"\n", + " \"and Raphael Warnock in a special runoff election for a two-year term. \"\n", + " \"President-elect Biden had supported and campaigned for both \"\n", + " \"candidates prior to the runoff elections on January 5.On January 6, \"\n", + " \"a mob of thousands of Trump supporters violently stormed the Capitol \"\n", + " \"in the hope of overturning Biden's election, forcing Congress to \"\n", + " \"evacuate during the counting of the Electoral College votes. More \"\n", + " \"than 26,000 National Guard members were deployed to the capital \"\n", + " \"for the inauguration, with thousands remaining into the spring.\"},\n", + "\n", + "{\"query\": \"What is the list of the top financial highlights for the quarter?\",\n", + " \"answer\": \"•Revenue: $52.9 million, up 10% in constant currency;\\n\"\n", + " \"•Operating income: $22.4 billion, up 15% in constant currency;\\n\"\n", + " \"•Net income: $18.3 billion, up 14% in constant currency;\\n\"\n", + " \"•Diluted earnings per share: $2.45 billion, up 14% in constant currency.\",\n", + " \"context\": \"Microsoft Cloud Strength Drives Third Quarter Results \\nREDMOND, Wash. — April 25, 2023 — \"\n", + " \"Microsoft Corp. today announced the following results for the quarter ended March 31, 2023,\"\n", + " \" as compared to the corresponding period of last fiscal year:\\n· Revenue was $52.9 billion\"\n", + " \" and increased 7% (up 10% in constant currency)\\n· Operating income was $22.4 billion \"\n", + " \"and increased 10% (up 15% in constant currency)\\n· Net income was $18.3 billion and \"\n", + " \"increased 9% (up 14% in constant currency)\\n· Diluted earnings per share was $2.45 \"\n", + " \"and increased 10% (up 14% in constant currency).\\n\"},\n", + "\n", + "{\"query\": \"What is a list of the key points?\",\n", + " \"answer\": \"•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in \"\n", + " \"Treasury yields;\\n•Dow Jones gained 195.12 points;\\n•S&P 500 added 1.59%;\\n•Nasdaq Composite rose \"\n", + " \"1.35%;\\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\\n\"\n", + " \"•10-year Treasury rate trading near the highest level in 14 years at 4.58%.\",\n", + " \"context\": \"Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data \"\n", + " \"and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, \"\n", + " \"or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy \"\n", + " \"Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in \"\n", + " \"August, the Labor Department said. Economists polled by Dow Jones expected 273,000 \"\n", + " \"jobs. However, wages rose less than expected last month. Stocks posted a stunning \"\n", + " \"turnaround on Friday, after initially falling on the stronger-than-expected jobs report. \"\n", + " \"At its session low, the Dow had fallen as much as 198 points; it surged by more than \"\n", + " \"500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during \"\n", + " \"their lowest points in the day. Traders were unclear of the reason for the intraday \"\n", + " \"reversal. Some noted it could be the softer wage number in the jobs report that made \"\n", + " \"investors rethink their earlier bearish stance. Others noted the pullback in yields from \"\n", + " \"the day’s highs. Part of the rally may just be to do a market that had gotten extremely \"\n", + " \"oversold with the S&P 500 at one point this week down more than 9% from its high earlier \"\n", + " \"this year. Yields initially surged after the report, with the 10-year Treasury rate trading \"\n", + " \"near its highest level in 14 years. The benchmark rate later eased from those levels, but \"\n", + " \"was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back \"\n", + " \"in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s \"\n", + " \"helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries \"\n", + " \"Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially \"\n", + " \"some oversold conditions.'\"}\n", + "\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 388, + "referenced_widgets": [ + "795a24acc52a4334a0f355cb33e7b841", + "7f5115a036524531b1a144c06adebb97", + "91e5e40e422f43a7ac092b0701f92cba", + "b42bb6a1b8844ff6a08ff243dedeb2c7", + "fa30ff08a49a4167baa76bf7cb594f50", + "4c7cd0ca4814454ab349d7867a1d8cab", + "fafb4c3ee8cd4781b2e6ebff9122fb11", + "9d1f5ad555604599833a5dcbd39d38c7", + "7d732fe7e8d14b7f911df38eb79ef50e", + "ede38c29bd7341ef92aca8eefd83ff15", + "c14a213cb14d4eba85ca4eec8a6f7961", + "f155c615e275447a83870d0b3acfad05", + "0908b8a5858f4e9baf97b71d83609b9b", + "e8bf9841cbd840efbd620f42ef55bc7c", + "31623eeb16da49bfbed0681627b3cae0", + "ae1313e57ff44dad83068bd754a7d479", + "2f4e36e1a6d945439acddb7026028d19", + "78bcb4c6ea254f108909d8051523c17b", + "6e6856b84a1e487b93bcdef5fce1a190", + "21ce79e851cb4e7ea247f433d73204b5", + "e742feb817f845e0a5709fb37540d541", + "e59148c840e74bcca98f878aceef74dd", + "7e65489c12f54b688117b4841497e009", + "e6f4fa682d3246398d9f98858b020a75", + "f3c31b94c820409dade607ad73b489c9", + "ff7a14a5ac6e47998303328896d85122", + "bafc77deb95049f7980a63ed26e83d64", + "005025f07eeb4aacb37c6d505dfd7065", + "6989643b74c0490cacc3708f666f2baf", + "97f1cbada54c4d8ead283f21dbacd7f2", + "c17e9115d13846b893204ce8c371e9f8", + "8d573c80a48e4b44aee94f65fe8d6dbc", + "50a73768b5c041f28ac4595ce850d97a", + "7d030a854a944287879bc1a8b79879a8", + "fce6e5ed3978484d9471b8aa9504fcb5", + "bb5de60fa62d4d449bcfda8a3e8aecef", + "30276c2e709e496d9e60a58dd6d73cb1", + "50c68373e0f94520b86be47ef8337e65", + "afbb8bf73f2e45959e2d65f08ce58441", + "3cc039d8b3304741a98ee0a498cd07c8", + "cc67aaf2edeb4a4d8b75e526ecf7079a", + "210c96ec80c645cf9ab6857d6eda04ac", + "e2c4add149414f9a990497a75a6a6aa3", + "006d79a0ab524b2ebce0d5dcefa48dda", + "9581442011bb49fea052252fd3769c2e", + "e0df690926bb40278b368a3c018f529e", + "0b63902f9884455e9a08705d8c7835b5", + "6518cd1e5ee848a18a6bcd8388157151", + "73ea88dd98d84e75a61c475d0f836b10", + "25cc1e4cfe954c08ae0808b2c09e0805", + "879f323ab7284aa7b8af1930289f9142", + "d10fd2224e5f4c31a112d917b87f4cf2", + "45911ab8466245628e5af793587159b7", + "fedd18c336ed4ae5a0774a514c07cc6c", + "fcfbcf368af34932a91d4f1f9f6f5304" + ] + }, + "id": "PnJ3mNCVGLY4", + "outputId": "3a417be6-3343-43a4-e810-5399d4ac57e5" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " > Loading Model: bling-answer-tool...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "795a24acc52a4334a0f355cb33e7b841", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00 Loading Model: {model_name}...\")\n", + "prompter = Prompt().load_model(model_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1XLuwODRGYvk", + "outputId": "16286067-5682-46cb-ff44-55ef94a9f435" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " > Model bling-answer-tool load time: 15.268526554107666\n" + ] + } + ], + "source": [ + "t1 = time.time()\n", + "print(f\"\\n > Model {model_name} load time: {t1-t0 }\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "d6jTsiAdNrPo", + "outputId": "20c25bda-92e1-4427-a381-41164fd08e44" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "1. Query: What is the total amount of the invoice?\n", + "LLM Response: 22,500.00\n", + "Gold Answer: $22,500.00\n", + "LLM Usage: {'input': 209, 'output': 9, 'total': 218, 'metric': 'tokens', 'processing_time': 2.0669240951538086}\n", + "\n", + "2. Query: What was the amount of the trade surplus?\n", + "LLM Response: 62.4 billion yen ($416.6 million)\n", + "Gold Answer: 62.4 billion yen ($416.6 million)\n", + "LLM Usage: {'input': 178, 'output': 15, 'total': 193, 'metric': 'tokens', 'processing_time': 1.638265609741211}\n", + "\n", + "3. Query: What was Microsoft's revenue in the 3rd quarter?\n", + "LLM Response: 52.9 billion, up 7% (up 10% in constant currency)\n", + "Gold Answer: $52.9 billion\n", + "LLM Usage: {'input': 203, 'output': 20, 'total': 223, 'metric': 'tokens', 'processing_time': 2.359246015548706}\n", + "\n", + "4. Query: When did the LISP machine market collapse?\n", + "LLM Response: 1987\n", + "Gold Answer: 1987.\n", + "LLM Usage: {'input': 435, 'output': 4, 'total': 439, 'metric': 'tokens', 'processing_time': 2.45987606048584}\n", + "\n", + "5. Query: When will employment start?\n", + "LLM Response: April 16, 2012\n", + "Gold Answer: April 16, 2012.\n", + "LLM Usage: {'input': 186, 'output': 11, 'total': 197, 'metric': 'tokens', 'processing_time': 0.8680579662322998}\n", + "\n", + "6. Query: What is the current rate on 10-year treasuries?\n", + "LLM Response: 6 basis points at 4.58%\n", + "Gold Answer: 4.58%\n", + "LLM Usage: {'input': 552, 'output': 10, 'total': 562, 'metric': 'tokens', 'processing_time': 2.0527071952819824}\n", + "\n", + "7. Query: What is the governing law?\n", + "LLM Response: State of Massachusetts\n", + "Gold Answer: State of Massachusetts\n", + "LLM Usage: {'input': 338, 'output': 3, 'total': 341, 'metric': 'tokens', 'processing_time': 1.1315374374389648}\n", + "\n", + "8. Query: What is the amount of the base salary?\n", + "LLM Response: 200,000\n", + "Gold Answer: $200,000.\n", + "LLM Usage: {'input': 200, 'output': 7, 'total': 207, 'metric': 'tokens', 'processing_time': 0.8254244327545166}\n", + "\n", + "9. Query: Is the expected gross margin greater than 70%?\n", + "LLM Response: Yes, 71.5% > 70%.\n", + "Gold Answer: Yes, between 71.5% and 72.%\n", + "LLM Usage: {'input': 339, 'output': 13, 'total': 352, 'metric': 'tokens', 'processing_time': 1.3918354511260986}\n", + "\n", + "10. Query: What is Bank of America's rating on Target?\n", + "LLM Response: TRADE: Buy\n", + "Gold Answer: Buy\n", + "LLM Usage: {'input': 787, 'output': 6, 'total': 793, 'metric': 'tokens', 'processing_time': 3.8873860836029053}\n", + "\n", + "11. Query: Who is NVIDIA's partner for the driver assistance system?\n", + "LLM Response: MediaTek\n", + "Gold Answer: MediaTek\n", + "LLM Usage: {'input': 136, 'output': 3, 'total': 139, 'metric': 'tokens', 'processing_time': 0.5586395263671875}\n", + "\n", + "12. Query: What was the rate of decline in 3rd quarter sales?\n", + "LLM Response: 20%\n", + "Gold Answer: 20% year-on-year.\n", + "LLM Usage: {'input': 149, 'output': 3, 'total': 152, 'metric': 'tokens', 'processing_time': 0.607762336730957}\n", + "\n", + "13. Query: What was professional visualization revenue in the quarter?\n", + "LLM Response: Not Found.\n", + "Gold Answer: $379 million\n", + "LLM Usage: {'input': 423, 'output': 3, 'total': 426, 'metric': 'tokens', 'processing_time': 1.4556689262390137}\n", + "\n", + "14. Query: What is the executive's title?\n", + "LLM Response: Senior Vice President, Event Planning\n", + "Gold Answer: Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.\n", + "LLM Usage: {'input': 410, 'output': 9, 'total': 419, 'metric': 'tokens', 'processing_time': 1.498265027999878}\n", + "\n", + "15. Query: According to the CFO, what led to the increase in cloud revenue?\n", + "LLM Response: 1. Focused execution by the sales teams and partners in this dynamic environment. 2. Microsoft Cloud revenue of $28.5 billion, up 22% (up 25% in constant currency). 3. Amy Hood, executive vice president and chief financial officer of Microsoft.\n", + "Gold Answer: Focused execution by our sales teams and partners\n", + "LLM Usage: {'input': 176, 'output': 67, 'total': 243, 'metric': 'tokens', 'processing_time': 2.3003711700439453}\n", + "\n", + "16. Query: Which company is located in Nevada?\n", + "LLM Response: North Industries\n", + "Gold Answer: North Industries\n", + "LLM Usage: {'input': 129, 'output': 4, 'total': 133, 'metric': 'tokens', 'processing_time': 0.5510938167572021}\n", + "\n", + "17. Query: When can termination after a material breach occur?\n", + "LLM Response: 30 days written notice\n", + "Gold Answer: If the breach is not cured within 15 days of notice of the breach.\n", + "LLM Usage: {'input': 162, 'output': 5, 'total': 167, 'metric': 'tokens', 'processing_time': 0.6938717365264893}\n", + "\n", + "18. Query: What is a headline summary in 10 words or less?\n", + "LLM Response: Joe Biden's tenure as the 46th president of the United States began with his inauguration on January 20, 2021.\n", + "Gold Answer: Joe Biden is the 46th President of the United States.\n", + "LLM Usage: {'input': 117, 'output': 36, 'total': 153, 'metric': 'tokens', 'processing_time': 1.4063897132873535}\n", + "\n", + "19. Query: Who are the two people that won elections in Georgia?\n", + "LLM Response: Jon Ossoff and Raphael Warnock\n", + "Gold Answer: Jon Ossoff and Raphael Warnock\n", + "LLM Usage: {'input': 365, 'output': 12, 'total': 377, 'metric': 'tokens', 'processing_time': 1.9793884754180908}\n", + "\n", + "20. Query: What is the list of the top financial highlights for the quarter?\n", + "LLM Response: •Revenue was $52.9 billion and increased 7% (up 10% in constant currency)\n", + "•Operating income was $22.4 billion and increased 10% (up 15% in constant currency)\n", + "•Net income was $18.3 billion and increased 9% (up 14% in constant currency)\n", + "•Diluted earnings per share was $2.45 and increased 10% (up 14% in constant currency)\n", + "Gold Answer: •Revenue: $52.9 million, up 10% in constant currency;\n", + "•Operating income: $22.4 billion, up 15% in constant currency;\n", + "•Net income: $18.3 billion, up 14% in constant currency;\n", + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.\n", + "LLM Usage: {'input': 203, 'output': 110, 'total': 313, 'metric': 'tokens', 'processing_time': 4.0941481590271}\n", + "\n", + "21. Query: What is a list of the key points?\n", + "LLM Response: •Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data and a major increase in Treasury yields.\n", + "•The Dow Jones Industrial Average gained 195.12 points, or 0.76%, to close at 31,419.58.\n", + "•The S&P 500 added 1.59% at 4,008.50.\n", + "•The tech-heavy Nasdaq Composite rose 1.35%, closing at 12,299.68.\n", + "•The U.S. economy added 438,000 jobs in August, the Labor Department said. Economists polled by Dow Jones expected 273,000 jobs. However, wages rose less than expected last month.\n", + "Gold Answer: •Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in Treasury yields;\n", + "•Dow Jones gained 195.12 points;\n", + "•S&P 500 added 1.59%;\n", + "•Nasdaq Composite rose 1.35%;\n", + "•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n", + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.\n", + "LLM Usage: {'input': 546, 'output': 189, 'total': 735, 'metric': 'tokens', 'processing_time': 7.0252697467803955}\n", + "\n", + "Total processing time: 40.96032643318176 seconds\n" + ] + } + ], + "source": [ + "for i, entries in enumerate(test_list):\n", + " print(f\"\\n{i+1}. Query: {entries['query']}\")\n", + "\n", + " output = prompter.prompt_main(entries[\"query\"],\n", + " context=entries[\"context\"],\n", + " prompt_name=\"default_with_context\",\n", + " temperature=0.30)\n", + "\n", + " llm_response = output[\"llm_response\"].strip(\"\\n\")\n", + " print(f\"LLM Response: {llm_response}\")\n", + "\n", + " print(f\"Gold Answer: {entries['answer']}\")\n", + "\n", + " print(f\"LLM Usage: {output['usage']}\")\n", + "\n", + "t2 = time.time()\n", + "print(f\"\\nTotal processing time: {t2-t1} seconds\")" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "005025f07eeb4aacb37c6d505dfd7065": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "006d79a0ab524b2ebce0d5dcefa48dda": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0908b8a5858f4e9baf97b71d83609b9b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2f4e36e1a6d945439acddb7026028d19", + "placeholder": "​", + "style": "IPY_MODEL_78bcb4c6ea254f108909d8051523c17b", + "value": ".gitattributes: 100%" + } + }, + "0b63902f9884455e9a08705d8c7835b5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d10fd2224e5f4c31a112d917b87f4cf2", + "max": 668787680, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_45911ab8466245628e5af793587159b7", + "value": 668787680 + } + }, + "210c96ec80c645cf9ab6857d6eda04ac": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "21ce79e851cb4e7ea247f433d73204b5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "25cc1e4cfe954c08ae0808b2c09e0805": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2f4e36e1a6d945439acddb7026028d19": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "30276c2e709e496d9e60a58dd6d73cb1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e2c4add149414f9a990497a75a6a6aa3", + "placeholder": "​", + "style": "IPY_MODEL_006d79a0ab524b2ebce0d5dcefa48dda", + "value": " 1.50k/1.50k [00:00<00:00, 75.2kB/s]" + } + }, + "31623eeb16da49bfbed0681627b3cae0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e742feb817f845e0a5709fb37540d541", + "placeholder": "​", + "style": "IPY_MODEL_e59148c840e74bcca98f878aceef74dd", + "value": " 1.62k/1.62k [00:00<00:00, 54.4kB/s]" + } + }, + "3cc039d8b3304741a98ee0a498cd07c8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "45911ab8466245628e5af793587159b7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4c7cd0ca4814454ab349d7867a1d8cab": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "50a73768b5c041f28ac4595ce850d97a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "50c68373e0f94520b86be47ef8337e65": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6518cd1e5ee848a18a6bcd8388157151": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fedd18c336ed4ae5a0774a514c07cc6c", + "placeholder": "​", + "style": "IPY_MODEL_fcfbcf368af34932a91d4f1f9f6f5304", + "value": " 669M/669M [00:04<00:00, 129MB/s]" + } + }, + "6989643b74c0490cacc3708f666f2baf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6e6856b84a1e487b93bcdef5fce1a190": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "73ea88dd98d84e75a61c475d0f836b10": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "78bcb4c6ea254f108909d8051523c17b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "795a24acc52a4334a0f355cb33e7b841": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7f5115a036524531b1a144c06adebb97", + "IPY_MODEL_91e5e40e422f43a7ac092b0701f92cba", + "IPY_MODEL_b42bb6a1b8844ff6a08ff243dedeb2c7" + ], + "layout": "IPY_MODEL_fa30ff08a49a4167baa76bf7cb594f50" + } + }, + "7d030a854a944287879bc1a8b79879a8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_fce6e5ed3978484d9471b8aa9504fcb5", + "IPY_MODEL_bb5de60fa62d4d449bcfda8a3e8aecef", + "IPY_MODEL_30276c2e709e496d9e60a58dd6d73cb1" + ], + "layout": "IPY_MODEL_50c68373e0f94520b86be47ef8337e65" + } + }, + "7d732fe7e8d14b7f911df38eb79ef50e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7e65489c12f54b688117b4841497e009": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e6f4fa682d3246398d9f98858b020a75", + "IPY_MODEL_f3c31b94c820409dade607ad73b489c9", + "IPY_MODEL_ff7a14a5ac6e47998303328896d85122" + ], + "layout": "IPY_MODEL_bafc77deb95049f7980a63ed26e83d64" + } + }, + "7f5115a036524531b1a144c06adebb97": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4c7cd0ca4814454ab349d7867a1d8cab", + "placeholder": "​", + "style": "IPY_MODEL_fafb4c3ee8cd4781b2e6ebff9122fb11", + "value": "Fetching 4 files: 100%" + } + }, + "879f323ab7284aa7b8af1930289f9142": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8d573c80a48e4b44aee94f65fe8d6dbc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "91e5e40e422f43a7ac092b0701f92cba": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9d1f5ad555604599833a5dcbd39d38c7", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_7d732fe7e8d14b7f911df38eb79ef50e", + "value": 4 + } + }, + "9581442011bb49fea052252fd3769c2e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e0df690926bb40278b368a3c018f529e", + "IPY_MODEL_0b63902f9884455e9a08705d8c7835b5", + "IPY_MODEL_6518cd1e5ee848a18a6bcd8388157151" + ], + "layout": "IPY_MODEL_73ea88dd98d84e75a61c475d0f836b10" + } + }, + "97f1cbada54c4d8ead283f21dbacd7f2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9d1f5ad555604599833a5dcbd39d38c7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ae1313e57ff44dad83068bd754a7d479": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "afbb8bf73f2e45959e2d65f08ce58441": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b42bb6a1b8844ff6a08ff243dedeb2c7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ede38c29bd7341ef92aca8eefd83ff15", + "placeholder": "​", + "style": "IPY_MODEL_c14a213cb14d4eba85ca4eec8a6f7961", + "value": " 4/4 [00:05<00:00,  1.98s/it]" + } + }, + "bafc77deb95049f7980a63ed26e83d64": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bb5de60fa62d4d449bcfda8a3e8aecef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cc67aaf2edeb4a4d8b75e526ecf7079a", + "max": 1495, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_210c96ec80c645cf9ab6857d6eda04ac", + "value": 1495 + } + }, + "c14a213cb14d4eba85ca4eec8a6f7961": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c17e9115d13846b893204ce8c371e9f8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cc67aaf2edeb4a4d8b75e526ecf7079a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d10fd2224e5f4c31a112d917b87f4cf2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e0df690926bb40278b368a3c018f529e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_25cc1e4cfe954c08ae0808b2c09e0805", + "placeholder": "​", + "style": "IPY_MODEL_879f323ab7284aa7b8af1930289f9142", + "value": "bling-answer.gguf: 100%" + } + }, + "e2c4add149414f9a990497a75a6a6aa3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e59148c840e74bcca98f878aceef74dd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e6f4fa682d3246398d9f98858b020a75": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_005025f07eeb4aacb37c6d505dfd7065", + "placeholder": "​", + "style": "IPY_MODEL_6989643b74c0490cacc3708f666f2baf", + "value": "config.json: 100%" + } + }, + "e742feb817f845e0a5709fb37540d541": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e8bf9841cbd840efbd620f42ef55bc7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6e6856b84a1e487b93bcdef5fce1a190", + "max": 1623, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_21ce79e851cb4e7ea247f433d73204b5", + "value": 1623 + } + }, + "ede38c29bd7341ef92aca8eefd83ff15": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f155c615e275447a83870d0b3acfad05": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0908b8a5858f4e9baf97b71d83609b9b", + "IPY_MODEL_e8bf9841cbd840efbd620f42ef55bc7c", + "IPY_MODEL_31623eeb16da49bfbed0681627b3cae0" + ], + "layout": "IPY_MODEL_ae1313e57ff44dad83068bd754a7d479" + } + }, + "f3c31b94c820409dade607ad73b489c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_97f1cbada54c4d8ead283f21dbacd7f2", + "max": 246794, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c17e9115d13846b893204ce8c371e9f8", + "value": 246794 + } + }, + "fa30ff08a49a4167baa76bf7cb594f50": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fafb4c3ee8cd4781b2e6ebff9122fb11": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fce6e5ed3978484d9471b8aa9504fcb5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_afbb8bf73f2e45959e2d65f08ce58441", + "placeholder": "​", + "style": "IPY_MODEL_3cc039d8b3304741a98ee0a498cd07c8", + "value": "README.md: 100%" + } + }, + "fcfbcf368af34932a91d4f1f9f6f5304": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fedd18c336ed4ae5a0774a514c07cc6c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ff7a14a5ac6e47998303328896d85122": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8d573c80a48e4b44aee94f65fe8d6dbc", + "placeholder": "​", + "style": "IPY_MODEL_50a73768b5c041f28ac4595ce850d97a", + "value": " 247k/247k [00:00<00:00, 1.29MB/s]" + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_3_prompts_and_models_version_2.ipynb b/tutorials/notebooks/fast_start_examples/example_3_prompts_and_models_version_2.ipynb new file mode 100644 index 0000000..c4cf0a2 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_3_prompts_and_models_version_2.ipynb @@ -0,0 +1,1948 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "pVMW5qfZGndc", + "outputId": "4dd85b9b-d9bd-4e11-bab6-c2232cd485c6" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: llmware in /usr/local/lib/python3.10/dist-packages (0.2.14)\n", + "Requirement already satisfied: boto3==1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.24.53)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.20.3)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Requirement already satisfied: openai>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.30.1)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.7.2)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from llmware) (2.2.1+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.40.2)\n", + "Requirement already satisfied: Wikipedia-API==0.6.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.6.0)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: einops==0.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.7.0)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: botocore<1.28.0,>=1.27.53 in /usr/local/lib/python3.10/dist-packages (from boto3==1.24.53->llmware) (1.27.96)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3==1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /usr/local/lib/python3.10/dist-packages (from boto3==1.24.53->llmware) (0.6.2)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.11.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Wikipedia-API==0.6.0->llmware) (2.31.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->llmware) (1.7.0)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (0.27.0)\n", + "Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (2.7.1)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (1.3.1)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (3.1.4)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (12.1.105)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (8.9.2.26)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (2.19.3)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (12.1.105)\n", + "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (2.2.0)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->llmware) (12.4.127)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware) (0.4.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (3.7)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (1.2.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.26.18)\n", + "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2024.2.2)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (1.0.5)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware) (0.14.0)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->llmware) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n", + "Collecting grpcio==1.60.0\n", + " Downloading grpcio-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hInstalling collected packages: grpcio\n", + " Attempting uninstall: grpcio\n", + " Found existing installation: grpcio 1.60.0\n", + " Uninstalling grpcio-1.60.0:\n", + " Successfully uninstalled grpcio-1.60.0\n", + "Successfully installed grpcio-1.60.0\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "%pip install \"grpcio==1.60.0\" --no-cache-dir --force-reinstall" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "gXok1vfMG2no" + }, + "outputs": [], + "source": [ + "import time\n", + "from llmware.prompts import Prompt\n", + "from llmware.models import ModelCatalog" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "_2WxmHPGLHmn" + }, + "outputs": [], + "source": [ + "test_list = [\n", + "\n", + "{\"query\": \"What is the total amount of the invoice?\",\n", + " \"answer\": \"$22,500.00\",\n", + " \"context\": \"Services Vendor Inc. \\n100 Elm Street Pleasantville, NY \\nTO Alpha Inc. 5900 1st Street \"\n", + " \"Los Angeles, CA \\nDescription Front End Engineering Service $5000.00 \\n Back End Engineering\"\n", + " \" Service $7500.00 \\n Quality Assurance Manager $10,000.00 \\n Total Amount $22,500.00 \\n\"\n", + " \"Make all checks payable to Services Vendor Inc. Payment is due within 30 days.\"\n", + " \"If you have any questions concerning this invoice, contact Bia Hermes. \"\n", + " \"THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000\"},\n", + "\n", + "{\"query\": \"What was the amount of the trade surplus?\",\n", + " \"answer\": \"62.4 billion yen ($416.6 million)\",\n", + " \"context\": \"Japan’s September trade balance swings into surplus, surprising expectations\"\n", + " \"Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, \"\n", + " \"beating expectations from economists polled by Reuters for a trade deficit of 42.5 \"\n", + " \"billion yen. Data from Japan’s customs agency revealed that exports in September \"\n", + " \"increased 4.3% year on year, while imports slid 16.3% compared to the same period \"\n", + " \"last year. According to FactSet, exports to Asia fell for the ninth straight month, \"\n", + " \"which reflected ongoing China weakness. Exports were supported by shipments to \"\n", + " \"Western markets, FactSet added. — Lim Hui Jie\"},\n", + "\n", + "{\"query\": \"What was Microsoft's revenue in the 3rd quarter?\",\n", + " \"answer\": \"$52.9 billion\",\n", + " \"context\": \"Microsoft Cloud Strength Drives Third Quarter Results \\nREDMOND, Wash. — April 25, 2023 — \"\n", + " \"Microsoft Corp. today announced the following results for the quarter ended March 31, 2023,\"\n", + " \" as compared to the corresponding period of last fiscal year:\\n· Revenue was $52.9 billion\"\n", + " \" and increased 7% (up 10% in constant currency)\\n· Operating income was $22.4 billion \"\n", + " \"and increased 10% (up 15% in constant currency)\\n· Net income was $18.3 billion and \"\n", + " \"increased 9% (up 14% in constant currency)\\n· Diluted earnings per share was $2.45 \"\n", + " \"and increased 10% (up 14% in constant currency).\\n\"},\n", + "\n", + "{\"query\": \"When did the LISP machine market collapse?\",\n", + " \"answer\": \"1987.\",\n", + " \"context\": \"The attendees became the leaders of AI research in the 1960s.\"\n", + " \" They and their students produced programs that the press described as 'astonishing': \"\n", + " \"computers were learning checkers strategies, solving word problems in algebra, \"\n", + " \"proving logical theorems and speaking English. By the middle of the 1960s, research in \"\n", + " \"the U.S. was heavily funded by the Department of Defense and laboratories had been \"\n", + " \"established around the world. Herbert Simon predicted, 'machines will be capable, \"\n", + " \"within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, \"\n", + " \"'within a generation ... the problem of creating 'artificial intelligence' will \"\n", + " \"substantially be solved'. They had, however, underestimated the difficulty of the problem. \"\n", + " \"Both the U.S. and British governments cut off exploratory research in response \"\n", + " \"to the criticism of Sir James Lighthill and ongoing pressure from the US Congress \"\n", + " \"to fund more productive projects. Minsky's and Papert's book Perceptrons was understood \"\n", + " \"as proving that artificial neural networks approach would never be useful for solving \"\n", + " \"real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period \"\n", + " \"when obtaining funding for AI projects was difficult, followed. In the early 1980s, \"\n", + " \"AI research was revived by the commercial success of expert systems, a form of AI \"\n", + " \"program that simulated the knowledge and analytical skills of human experts. By 1985, \"\n", + " \"the market for AI had reached over a billion dollars. At the same time, Japan's fifth \"\n", + " \"generation computer project inspired the U.S. and British governments to restore funding \"\n", + " \"for academic research. However, beginning with the collapse of the Lisp Machine market \"\n", + " \"in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began.\"},\n", + "\n", + "{\"query\": \"When will employment start?\",\n", + " \"answer\": \"April 16, 2012.\",\n", + " \"context\": \"THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered \"\n", + " \"into this 2nd day of April, 2012, by and between Aphrodite Apollo \"\n", + " \"(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), \"\n", + " \"and shall become effective upon Executive’s commencement of employment \"\n", + " \"(the “Effective Date”) which is expected to commence on April 16, 2012. \"\n", + " \"The Company and Executive agree that unless Executive has commenced \"\n", + " \"employment with the Company as of April 16, 2012 (or such later date as \"\n", + " \"agreed by each of the Company and Executive) this Agreement shall be \"\n", + " \"null and void and of no further effect.\"},\n", + "\n", + "{\"query\": \"What is the current rate on 10-year treasuries?\",\n", + " \"answer\": \"4.58%\",\n", + " \"context\": \"Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data \"\n", + " \"and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, \"\n", + " \"or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy \"\n", + " \"Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in \"\n", + " \"August, the Labor Department said. Economists polled by Dow Jones expected 273,000 \"\n", + " \"jobs. However, wages rose less than expected last month. Stocks posted a stunning \"\n", + " \"turnaround on Friday, after initially falling on the stronger-than-expected jobs report. \"\n", + " \"At its session low, the Dow had fallen as much as 198 points; it surged by more than \"\n", + " \"500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during \"\n", + " \"their lowest points in the day. Traders were unclear of the reason for the intraday \"\n", + " \"reversal. Some noted it could be the softer wage number in the jobs report that made \"\n", + " \"investors rethink their earlier bearish stance. Others noted the pullback in yields from \"\n", + " \"the day’s highs. Part of the rally may just be to do a market that had gotten extremely \"\n", + " \"oversold with the S&P 500 at one point this week down more than 9% from its high earlier \"\n", + " \"this year. Yields initially surged after the report, with the 10-year Treasury rate trading \"\n", + " \"near its highest level in 14 years. The benchmark rate later eased from those levels, but \"\n", + " \"was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back \"\n", + " \"in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s \"\n", + " \"helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries \"\n", + " \"Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially \"\n", + " \"some oversold conditions.'\"},\n", + "\n", + "{\"query\": \"What is the governing law?\",\n", + " \"answer\": \"State of Massachusetts\",\n", + " \"context\": \"19.\tGoverning Law and Procedures. This Agreement shall be governed by and interpreted \"\n", + " \"under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement,\"\n", + " \" which shall be governed by the laws of the State of Delaware, without giving effect to any \"\n", + " \"conflict of laws provisions. Employer and Executive each irrevocably and unconditionally \"\n", + " \"(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief \"\n", + " \"or other equitable relief related to this Agreement or any action commenced by Executive pursuant \"\n", + " \"to any provision hereof, may be brought in the United States District Court for the federal \"\n", + " \"district in which Executive’s principal place of employment is located, or if such court does \"\n", + " \"not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction \"\n", + " \"in the state and county in which Executive’s principal place of employment is located, \"\n", + " \"(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o\"\n", + " \"r proceeding, and (c) waives any objection which Employer or Executive may have to the \"\n", + " \"laying of venue of any such suit, action or proceeding in any such court. Employer and \"\n", + " \"Executive each also irrevocably and unconditionally consents to the service of any process, \"\n", + " \"pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8.\"},\n", + "\n", + "{\"query\": \"What is the amount of the base salary?\",\n", + " \"answer\": \"$200,000.\",\n", + " \"context\": \"2.2. Base Salary. For all the services rendered by Executive hereunder, during the \"\n", + " \"Employment Period, Employer shall pay Executive a base salary at the annual rate of \"\n", + " \"$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. \"\n", + " \"Executive’s base salary shall be reviewed annually by the Board (or the compensation committee \"\n", + " \"of the Board), pursuant to Employer’s normal compensation and performance review policies \"\n", + " \"for senior level executives, and may be increased but not decreased. The amount of any \"\n", + " \"increase for each year shall be determined accordingly. For purposes of this Agreement, \"\n", + " \"the term “Base Salary” shall mean the amount of Executive’s base salary established \"\n", + " \"from time to time pursuant to this Section 2.2. \"},\n", + "\n", + "{\"query\": \"Is the expected gross margin greater than 70%?\",\n", + " \"answer\": \"Yes, between 71.5% and 72.%\",\n", + " \"context\": \"Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:\"\n", + " \"Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP \"\n", + " \"gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus \"\n", + " \"50 basis points. GAAP and non-GAAP operating expenses are expected to be \"\n", + " \"approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP \"\n", + " \"other income and expense are expected to be an income of approximately $100 \"\n", + " \"million, excluding gains and losses from non-affiliated investments. GAAP and \"\n", + " \"non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items.\"\n", + " \"Highlights NVIDIA achieved progress since its previous earnings announcement \"\n", + " \"in these areas: Data Center Second-quarter revenue was a record $10.32 billion, \"\n", + " \"up 141% from the previous quarter and up 171% from a year ago. Announced that the \"\n", + " \"NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping \"\n", + " \"this quarter, with a second-generation version with HBM3e memory expected to ship \"\n", + " \"in Q2 of calendar 2024. \"},\n", + "\n", + "{\"query\": \"What is Bank of America's rating on Target?\",\n", + " \"answer\": \"Buy\",\n", + " \"context\": \"Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from \"\n", + " \"my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom \"\n", + " \"of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index \"\n", + " \"soared more than 22%. Hotter than expected September consumer price index, consumer \"\n", + " \"inflation. The Social Security Administration issues announced a 3.2% cost-of-living \"\n", + " \"adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. \"\n", + " \"Cites consumer price index showing sticky retail inflation for the fourth time \"\n", + " \"in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites \"\n", + " \"risk/reward from depressed levels. Traffic could improve. Gross margin upside. \"\n", + " \"Merchandising better. Freight and transportation better. Target to report quarter \"\n", + " \"next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), \"\n", + " \"the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs \"\n", + " \"tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, \"\n", + " \"Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating.\"\n", + " \"If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the \"\n", + " \"Market email newsletter for free. Barclays cuts price targets on consumer products: \"\n", + " \"UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from \"\n", + " \"$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. \"\n", + " \"Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers\"\n", + " \"(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek\"\n", + " \"(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on \"\n", + " \"third quarter of 19-cent per share drag on earnings. The buyer: investors led by \"\n", + " \"private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for \"\n", + " \"Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share \"\n", + " \"from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps \"\n", + " \"overweight (buy) rating but lowers price target to $139 per share from $150. \"\n", + " \"Sees “still challenging” environment into third-quarter print. The Club owns shares \"\n", + " \"in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) \"\n", + " \"to overweight from equal weight (buy from hold) but lowers price target to $224 per \"\n", + " \"share from $230. Risk reward upgrade. Best visibility of utility scale names.\"},\n", + "\n", + "{\"query\": \"Who is NVIDIA's partner for the driver assistance system?\",\n", + " \"answer\": \"MediaTek\",\n", + " \"context\": \"Automotive Second-quarter revenue was $253 million, down 15% from the previous \"\n", + " \"quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering \"\n", + " \"the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. \"\n", + " \"Partnered with MediaTek, which will develop mainstream automotive systems on \"\n", + " \"chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics.\"},\n", + "\n", + "{\"query\": \"What was the rate of decline in 3rd quarter sales?\",\n", + " \"answer\": \"20% year-on-year.\",\n", + " \"context\": \"Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following \"\n", + " \"third quarter earnings that plunged. The Finnish telecommunications giant said that \"\n", + " \"it will reduce its cost base and increase operation efficiency to “address the \"\n", + " \"challenging market environment. The substantial layoffs come after Nokia reported \"\n", + " \"third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over \"\n", + " \"the period plunged by 69% year-on-year to 133 million euros.\"},\n", + "\n", + "{\"query\": \"What was professional visualization revenue in the quarter?\",\n", + " \"answer\": \"$379 million\",\n", + " \"context\": \"Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up \"\n", + " \"22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, \"\n", + " \"bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299.\"\n", + " \"Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model \"\n", + " \"foundry service using AI-powered natural language interactions to transform games \"\n", + " \"by bringing intelligence to non-playable characters. Added 35 DLSS games, including \"\n", + " \"Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: \"\n", + " \"Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool.\"\n", + " \"Professional Visualization Second-quarter revenue was $379 million, up 28% from the \"\n", + " \"previous quarter and down 24% from a year ago. Announced three new desktop \"\n", + " \"workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 \"\n", + " \"and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are \"\n", + " \"shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, \"\n", + " \"with new foundation applications and services for developers and industrial \"\n", + " \"enterprises to optimize and enhance their 3D pipelines with OpenUSD and \"\n", + " \"generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the \"\n", + " \"Alliance for OpenUSD to promote the standardization, development, evolution and \"\n", + " \"growth of Universal Scene Description technology.\"},\n", + "\n", + "\n", + "{\"query\": \"What is the executive's title?\",\n", + " \"answer\": \"Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.\",\n", + " \"context\": \"2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, \"\n", + " \"Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s \"\n", + " \"Workforce Optimization Division. In such role, Executive will report to the Board of \"\n", + " \"Directors of Employer (the “Board”) and shall devote substantially all of his business time \"\n", + " \"and attention and his best efforts and ability to the operations of Employer and its subsidiaries. \"\n", + " \"Executive shall be responsible for running Employer’s day-to-day operations and shall perform \"\n", + " \"faithfully, diligently and competently the duties and responsibilities of a SVP and such other \"\n", + " \"duties and responsibilities as directed by the Board and are consistent with such position. \"\n", + " \"The foregoing shall not be construed as preventing Executive from (a) making passive \"\n", + " \"investments in other businesses or enterprises consistent with Employer’s code of conduct, \"\n", + " \"or (b) engaging in any other business activity consistent with Employer’s code of conduct; \"\n", + " \"provided that Executive seeks and obtains the prior approval of the Board before engaging \"\n", + " \"in any other business activity. In addition, it shall not be a violation of this Agreement \"\n", + " \"for Executive to participate in civic or charitable activities, deliver lectures, fulfill \"\n", + " \"speaking engagements, teach at educational institutions, and/or manage personal investments \"\n", + " \"(subject to the immediately preceding sentence); provided that such activities do not \"\n", + " \"interfere in any substantial respect with the performance of Executive’s responsibilities \"\n", + " \"as an employee in accordance with this Agreement. Executive may also serve on one or more \"\n", + " \"corporate boards of another company (and committees thereof) upon giving advance notice \"\n", + " \"to the Board prior to commencing service on any other corporate board.\"},\n", + "\n", + "{\"query\": \"According to the CFO, what led to the increase in cloud revenue?\",\n", + " \"answer\": \"Focused execution by our sales teams and partners\",\n", + " \"context\": \"'The world's most advanced AI models \"\n", + " \"are coming together with the world's most universal user interface - natural language - \"\n", + " \"to create a new era of computing,' said Satya Nadella, chairman and chief \"\n", + " \"executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform \"\n", + " \"of choice to help customers get the most value out of their digital spend and innovate \"\n", + " \"for this next generation of AI.' 'Focused execution by our sales teams and partners \"\n", + " \"in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, \"\n", + " \"up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive \"\n", + " \"vice president and chief financial officer of Microsoft.\\n\"},\n", + "\n", + "{\"query\": \"Which company is located in Nevada?\",\n", + " \"answer\": \"North Industries\",\n", + " \"context\": \"To send notices to Blue Moon Tech, mail to their headquarters at: \"\n", + " \"555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to\"\n", + " \"their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\\nTo send notices \"\n", + " \"to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234.\"},\n", + "\n", + "{\"query\": \"When can termination after a material breach occur?\",\n", + " \"answer\": \"If the breach is not cured within 15 days of notice of the breach.\",\n", + " \"context\": \"This Agreement shall remain in effect until terminated. Either party may terminate this \"\n", + " \"agreement, any Statement of Work or Services Description for convenience by giving the other \"\n", + " \"party 30 days written notice. Either party may terminate this Agreement or any work order or \"\n", + " \"services description if the other party is in material breach or default of any obligation \"\n", + " \"that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees \"\n", + " \"for services performed and expenses incurred prior to the termination of this Agreement. \"\n", + " \"Termination of this Agreement will terminate all outstanding Statement of Work or Services \"\n", + " \"Description entered into under this agreement.\"},\n", + "\n", + "{\"query\": \"What is a headline summary in 10 words or less?\",\n", + " \"answer\": \"Joe Biden is the 46th President of the United States.\",\n", + " \"context\": \"Joe Biden's tenure as the 46th president of the United States began with \"\n", + " \"his inauguration on January 20, 2021. Biden, a Democrat from Delaware who \"\n", + " \"previously served as vice president under Barack Obama, \"\n", + " \"took office following his victory in the 2020 presidential election over \"\n", + " \"Republican incumbent president Donald Trump. Upon his inauguration, he \"\n", + " \"became the oldest president in American history.\"},\n", + "\n", + "{\"query\": \"Who are the two people that won elections in Georgia?\",\n", + " \"answer\": \"Jon Ossoff and Raphael Warnock\",\n", + " \"context\": \"Though Biden was generally acknowledged as the winner, \"\n", + " \"General Services Administration head Emily W. Murphy \"\n", + " \"initially refused to begin the transition to the president-elect, \"\n", + " \"thereby denying funds and office space to his team. \"\n", + " \"On November 23, after Michigan certified its results, Murphy \"\n", + " \"issued the letter of ascertainment, granting the Biden transition \"\n", + " \"team access to federal funds and resources for an orderly transition. \"\n", + " \"Two days after becoming the projected winner of the 2020 election, \"\n", + " \"Biden announced the formation of a task force to advise him on the \"\n", + " \"COVID-19 pandemic during the transition, co-chaired by former \"\n", + " \"Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, \"\n", + " \"and Yale University's Marcella Nunez-Smith. On January 5, 2021, \"\n", + " \"the Democratic Party won control of the United States Senate, \"\n", + " \"effective January 20, as a result of electoral victories in \"\n", + " \"Georgia by Jon Ossoff in a runoff election for a six-year term \"\n", + " \"and Raphael Warnock in a special runoff election for a two-year term. \"\n", + " \"President-elect Biden had supported and campaigned for both \"\n", + " \"candidates prior to the runoff elections on January 5.On January 6, \"\n", + " \"a mob of thousands of Trump supporters violently stormed the Capitol \"\n", + " \"in the hope of overturning Biden's election, forcing Congress to \"\n", + " \"evacuate during the counting of the Electoral College votes. More \"\n", + " \"than 26,000 National Guard members were deployed to the capital \"\n", + " \"for the inauguration, with thousands remaining into the spring.\"},\n", + "\n", + "{\"query\": \"What is the list of the top financial highlights for the quarter?\",\n", + " \"answer\": \"•Revenue: $52.9 million, up 10% in constant currency;\\n\"\n", + " \"•Operating income: $22.4 billion, up 15% in constant currency;\\n\"\n", + " \"•Net income: $18.3 billion, up 14% in constant currency;\\n\"\n", + " \"•Diluted earnings per share: $2.45 billion, up 14% in constant currency.\",\n", + " \"context\": \"Microsoft Cloud Strength Drives Third Quarter Results \\nREDMOND, Wash. — April 25, 2023 — \"\n", + " \"Microsoft Corp. today announced the following results for the quarter ended March 31, 2023,\"\n", + " \" as compared to the corresponding period of last fiscal year:\\n· Revenue was $52.9 billion\"\n", + " \" and increased 7% (up 10% in constant currency)\\n· Operating income was $22.4 billion \"\n", + " \"and increased 10% (up 15% in constant currency)\\n· Net income was $18.3 billion and \"\n", + " \"increased 9% (up 14% in constant currency)\\n· Diluted earnings per share was $2.45 \"\n", + " \"and increased 10% (up 14% in constant currency).\\n\"},\n", + "\n", + "{\"query\": \"What is a list of the key points?\",\n", + " \"answer\": \"•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in \"\n", + " \"Treasury yields;\\n•Dow Jones gained 195.12 points;\\n•S&P 500 added 1.59%;\\n•Nasdaq Composite rose \"\n", + " \"1.35%;\\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\\n\"\n", + " \"•10-year Treasury rate trading near the highest level in 14 years at 4.58%.\",\n", + " \"context\": \"Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data \"\n", + " \"and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, \"\n", + " \"or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy \"\n", + " \"Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in \"\n", + " \"August, the Labor Department said. Economists polled by Dow Jones expected 273,000 \"\n", + " \"jobs. However, wages rose less than expected last month. Stocks posted a stunning \"\n", + " \"turnaround on Friday, after initially falling on the stronger-than-expected jobs report. \"\n", + " \"At its session low, the Dow had fallen as much as 198 points; it surged by more than \"\n", + " \"500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during \"\n", + " \"their lowest points in the day. Traders were unclear of the reason for the intraday \"\n", + " \"reversal. Some noted it could be the softer wage number in the jobs report that made \"\n", + " \"investors rethink their earlier bearish stance. Others noted the pullback in yields from \"\n", + " \"the day’s highs. Part of the rally may just be to do a market that had gotten extremely \"\n", + " \"oversold with the S&P 500 at one point this week down more than 9% from its high earlier \"\n", + " \"this year. Yields initially surged after the report, with the 10-year Treasury rate trading \"\n", + " \"near its highest level in 14 years. The benchmark rate later eased from those levels, but \"\n", + " \"was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back \"\n", + " \"in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s \"\n", + " \"helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries \"\n", + " \"Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially \"\n", + " \"some oversold conditions.'\"}\n", + "\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "kdQK_b-DOmC3" + }, + "outputs": [], + "source": [ + "# Step 1 - we will pick a model from the ModelCatalog\n", + "\n", + "# A few useful methods to discover and display a list of available models...\n", + "\n", + "# all generative models\n", + "llm_models = ModelCatalog().list_generative_models()\n", + "\n", + "# if you only want to see the local models\n", + "llm_local_models = ModelCatalog().list_generative_local_models()\n", + "\n", + "# to see only the open source models\n", + "llm_open_source_models = ModelCatalog().list_open_source_models()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "eyas9IWXStyC" + }, + "outputs": [], + "source": [ + "# for purposes of demo, try a few selected models from the list\n", + "\n", + "model_short_list = [\"llmware/bling-1b-0.1\",\n", + " \"llmware/bling-tiny-llama-v0\",\n", + " \"llmware/dragon-yi-6b-gguf\",\n", + " \"llmware/bling-falcon-1b-0.1\"]\n", + "\n", + "model_name = \"llmware/bling-1b-0.1\"\n", + "\n", + "model_name = \"llmware/bling-1b-0.1\"" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MgsC9kelSJP1", + "outputId": "6b7c4fa6-bc5e-46a3-dae0-58fcdbc5873e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "models: 0 Meta-Llama-3-8B HFGenerativeModel\n", + "models: 1 Meta-Llama-3-8B-Instruct HFGenerativeModel\n", + "models: 2 QuantFactory/Meta-Llama-3-8B-GGUF GGUFGenerativeModel\n", + "models: 3 QuantFactory/Meta-Llama-3-8B-Instruct-GGUF GGUFGenerativeModel\n", + "models: 4 TheBloke/Llama-2-7B-Chat-GGUF GGUFGenerativeModel\n", + "models: 5 TheBloke/OpenHermes-2.5-Mistral-7B-GGUF GGUFGenerativeModel\n", + "models: 6 TheBloke/Starling-LM-7B-alpha-GGUF GGUFGenerativeModel\n", + "models: 7 TheBloke/zephyr-7B-beta-GGUF GGUFGenerativeModel\n", + "models: 8 bartowski/Meta-Llama-3-8B-Instruct-GGUF GGUFGenerativeModel\n", + "models: 9 bling-answer-tool GGUFGenerativeModel\n", + "models: 10 bling-phi-3-gguf GGUFGenerativeModel\n", + "models: 11 bling-stablelm-3b-tool GGUFGenerativeModel\n", + "models: 12 dragon-llama-answer-tool GGUFGenerativeModel\n", + "models: 13 dragon-mistral-answer-tool GGUFGenerativeModel\n", + "models: 14 dragon-yi-answer-tool GGUFGenerativeModel\n", + "models: 15 llmware/bling-1.4b-0.1 HFGenerativeModel\n", + "models: 16 llmware/bling-1b-0.1 HFGenerativeModel\n", + "models: 17 llmware/bling-cerebras-1.3b-0.1 HFGenerativeModel\n", + "models: 18 llmware/bling-falcon-1b-0.1 HFGenerativeModel\n", + "models: 19 llmware/bling-phi-3 HFGenerativeModel\n", + "models: 20 llmware/bling-red-pajamas-3b-0.1 HFGenerativeModel\n", + "models: 21 llmware/bling-sheared-llama-1.3b-0.1 HFGenerativeModel\n", + "models: 22 llmware/bling-sheared-llama-2.7b-0.1 HFGenerativeModel\n", + "models: 23 llmware/bling-stable-lm-3b-4e1t-v0 HFGenerativeModel\n", + "models: 24 llmware/bling-tiny-llama-v0 HFGenerativeModel\n", + "models: 25 llmware/dragon-deci-6b-v0 HFGenerativeModel\n", + "models: 26 llmware/dragon-deci-7b-v0 HFGenerativeModel\n", + "models: 27 llmware/dragon-falcon-7b-v0 HFGenerativeModel\n", + "models: 28 llmware/dragon-llama-7b-gguf GGUFGenerativeModel\n", + "models: 29 llmware/dragon-llama-7b-v0 HFGenerativeModel\n", + "models: 30 llmware/dragon-mistral-7b-gguf GGUFGenerativeModel\n", + "models: 31 llmware/dragon-mistral-7b-v0 HFGenerativeModel\n", + "models: 32 llmware/dragon-red-pajama-7b-v0 HFGenerativeModel\n", + "models: 33 llmware/dragon-stablelm-7b-v0 HFGenerativeModel\n", + "models: 34 llmware/dragon-yi-6b-gguf GGUFGenerativeModel\n", + "models: 35 llmware/dragon-yi-6b-v0 HFGenerativeModel\n", + "models: 36 llmware/slim-category HFGenerativeModel\n", + "models: 37 llmware/slim-emotions HFGenerativeModel\n", + "models: 38 llmware/slim-intent HFGenerativeModel\n", + "models: 39 llmware/slim-ner HFGenerativeModel\n", + "models: 40 llmware/slim-nli HFGenerativeModel\n", + "models: 41 llmware/slim-ratings HFGenerativeModel\n", + "models: 42 llmware/slim-sentiment HFGenerativeModel\n", + "models: 43 llmware/slim-sql-1b-v0 HFGenerativeModel\n", + "models: 44 llmware/slim-tags HFGenerativeModel\n", + "models: 45 llmware/slim-topics HFGenerativeModel\n", + "models: 46 microsoft/Phi-3-mini-128k-instruct HFGenerativeModel\n", + "models: 47 microsoft/Phi-3-mini-4k-instruct HFGenerativeModel\n", + "models: 48 microsoft/Phi-3-mini-4k-instruct-gguf GGUFGenerativeModel\n", + "models: 49 slim-boolean HFGenerativeModel\n", + "models: 50 slim-boolean-tool GGUFGenerativeModel\n", + "models: 51 slim-category-tool GGUFGenerativeModel\n", + "models: 52 slim-emotions-tool GGUFGenerativeModel\n", + "models: 53 slim-extract HFGenerativeModel\n", + "models: 54 slim-extract-tool GGUFGenerativeModel\n", + "models: 55 slim-intent-tool GGUFGenerativeModel\n", + "models: 56 slim-ner-tool GGUFGenerativeModel\n", + "models: 57 slim-nli-tool GGUFGenerativeModel\n", + "models: 58 slim-ratings-tool GGUFGenerativeModel\n", + "models: 59 slim-sa-ner HFGenerativeModel\n", + "models: 60 slim-sa-ner-tool GGUFGenerativeModel\n", + "models: 61 slim-sentiment-tool GGUFGenerativeModel\n", + "models: 62 slim-sql-tool GGUFGenerativeModel\n", + "models: 63 slim-summary HFGenerativeModel\n", + "models: 64 slim-summary-tool GGUFGenerativeModel\n", + "models: 65 slim-tags-3b HFGenerativeModel\n", + "models: 66 slim-tags-3b-tool GGUFGenerativeModel\n", + "models: 67 slim-tags-tool GGUFGenerativeModel\n", + "models: 68 slim-topics-tool GGUFGenerativeModel\n", + "models: 69 slim-xsum HFGenerativeModel\n", + "models: 70 slim-xsum-tool GGUFGenerativeModel\n", + "models: 71 tiny-llama-chat-gguf GGUFGenerativeModel\n", + "models: 72 whisper-cpp-base WhisperCPPModel\n", + "models: 73 whisper-cpp-base-english WhisperCPPModel\n", + "models: 74 whisper-cpp-tiny-diarize WhisperCPPModel\n" + ] + } + ], + "source": [ + "# we will print out the local models\n", + "for i, models in enumerate(llm_local_models):\n", + " print(\"models: \", i, models[\"model_name\"], models[\"model_family\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "e46164dbf38e4f929942405eee41746b", + "96687e6808d54377a8bb8d5a0cb9e6ff", + "2d5d40b7513348bba66e6bf57da39e60", + "99bad8042d254c6792d40a3444db1ce9", + "ba44a68fc00944dd844cac4ea9c2f507", + "81f9f227f2aa47bfb007f2a7c1598bfa", + "31e2c815ea164d888c5276310fd8ddfb", + "858187f2ed144dc59ee5cd70d49a09c4", + "726d4b1cafad4c84898b9bbeb31bb21b", + "952644e2c51845c49b5093498c087c33", + "82fedb4609f84ed09b5f0e578b73875c", + "ea33d6532d7f4ad8af85c40588162e5f", + "4230b717baef43eaa0c4a7ca4aad207f", + "9c8fafd21215457390a63c5ff6a36a50", + "042b0f51652c4104b32c9aae84cc3815", + "24e0c0e9e25b417a8f6f35a952eba293", + "74d20e36afe740839b19b913404bb617", + "ababdb41145e4524babd99a4d061f375", + "9afd954348c342d1ac047cc78f544bbe", + "b0b170d1214541b18ee333ac1709061b", + "2095638235c04b0082d5975a6a173776", + "b00019ae3408427085d30ae245044562", + "26ca04a2a9be4232bd5ba979d280ef8d", + "6dd1fc3a33c5428580836442e77b8b8c", + "99b9def487bc4cbd8723a34865d36487", + "ce9e2fca4c334772a6fae7936454f530", + "c6e421b0df86447ea47a08132539e200", + "c1b3b62618084dc4a9c3451a081cc3d8", + "e7abfcc177204428b5891ddfe64877db", + "ff04ff6ce4f24e2f956fa8870079b012", + "41a1a662e00a494296c18901026f75ae", + "16960e9d85944511a4e43d03f1acec1b", + "cc779a09790d4186a6832dae389af0bf" + ] + }, + "id": "Fq0WeKMkS_Dv", + "outputId": "39a426bd-fe41-4721-b79e-d76910234b9c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " > Loading Model: llmware/bling-1b-0.1...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "e46164dbf38e4f929942405eee41746b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "config.json: 0%| | 0.00/2.27k [00:00 Model llmware/bling-1b-0.1 load time: 88.03528332710266 seconds\n", + "\n", + "1. Query: What is the total amount of the invoice?\n", + "LLM Response: $22,500.00\n", + "Gold Answer: $22,500.00\n", + "LLM Usage: {'input': 157, 'output': 7, 'total': 164, 'metric': 'tokens', 'processing_time': 10.29619574546814}\n", + "\n", + "2. Query: What was the amount of the trade surplus?\n", + "LLM Response: 62.4 billion yen\n", + "Gold Answer: 62.4 billion yen ($416.6 million)\n", + "LLM Usage: {'input': 147, 'output': 7, 'total': 154, 'metric': 'tokens', 'processing_time': 7.646324157714844}\n", + "\n", + "3. Query: What was Microsoft's revenue in the 3rd quarter?\n", + "LLM Response: $52.9 billion and increased 7% (up 10% in constant currency)\n", + "Gold Answer: $52.9 billion\n", + "LLM Usage: {'input': 162, 'output': 18, 'total': 180, 'metric': 'tokens', 'processing_time': 13.457484483718872}\n", + "\n", + "4. Query: When did the LISP machine market collapse?\n", + "LLM Response: 1987\n", + "Gold Answer: 1987.\n", + "LLM Usage: {'input': 368, 'output': 2, 'total': 370, 'metric': 'tokens', 'processing_time': 13.965656518936157}\n", + "\n", + "5. Query: When will employment start?\n", + "LLM Response: April 16, 2012\n", + "Gold Answer: April 16, 2012.\n", + "LLM Usage: {'input': 149, 'output': 5, 'total': 154, 'metric': 'tokens', 'processing_time': 8.105394124984741}\n", + "\n", + "6. Query: What is the current rate on 10-year treasuries?\n", + "LLM Response: 4.58%\n", + "Gold Answer: 4.58%\n", + "LLM Usage: {'input': 460, 'output': 5, 'total': 465, 'metric': 'tokens', 'processing_time': 17.908035039901733}\n", + "\n", + "7. Query: What is the governing law?\n", + "LLM Response: State of Massachusetts\n", + "Gold Answer: State of Massachusetts\n", + "LLM Usage: {'input': 289, 'output': 4, 'total': 293, 'metric': 'tokens', 'processing_time': 11.901292562484741}\n", + "\n", + "8. Query: What is the amount of the base salary?\n", + "LLM Response: $200,000\n", + "Gold Answer: $200,000.\n", + "LLM Usage: {'input': 170, 'output': 5, 'total': 175, 'metric': 'tokens', 'processing_time': 8.857550144195557}\n", + "\n", + "9. Query: Is the expected gross margin greater than 70%?\n", + "LLM Response: Yes, the expected gross margin is greater than 70% and the expected operating expenses are approximately $2.95 billion and $2.00 billion, respectively.\n", + "Gold Answer: Yes, between 71.5% and 72.%\n", + "LLM Usage: {'input': 278, 'output': 33, 'total': 311, 'metric': 'tokens', 'processing_time': 22.633046627044678}\n", + "\n", + "10. Query: What is Bank of America's rating on Target?\n", + "LLM Response: Not Found.\n", + "Gold Answer: Buy\n", + "LLM Usage: {'input': 654, 'output': 4, 'total': 658, 'metric': 'tokens', 'processing_time': 24.690168857574463}\n", + "\n", + "11. Query: Who is NVIDIA's partner for the driver assistance system?\n", + "LLM Response: MediaTek\n", + "Gold Answer: MediaTek\n", + "LLM Usage: {'input': 115, 'output': 4, 'total': 119, 'metric': 'tokens', 'processing_time': 6.630413293838501}\n", + "\n", + "12. Query: What was the rate of decline in 3rd quarter sales?\n", + "LLM Response: 20% year-on-year to 4.98 billion euros.\n", + "Gold Answer: 20% year-on-year.\n", + "LLM Usage: {'input': 116, 'output': 15, 'total': 131, 'metric': 'tokens', 'processing_time': 9.548069953918457}\n", + "\n", + "13. Query: What was professional visualization revenue in the quarter?\n", + "LLM Response: $379 million, up 28% from the previous quarter and down 24% from a year ago.\n", + "Gold Answer: $379 million\n", + "LLM Usage: {'input': 354, 'output': 21, 'total': 375, 'metric': 'tokens', 'processing_time': 20.629887342453003}\n", + "\n", + "14. Query: What is the executive's title?\n", + "LLM Response: SVP, Event Planning\n", + "Gold Answer: Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.\n", + "LLM Usage: {'input': 347, 'output': 6, 'total': 353, 'metric': 'tokens', 'processing_time': 14.70188856124878}\n", + "\n", + "15. Query: According to the CFO, what led to the increase in cloud revenue?\n", + "LLM Response: •Microsoft Cloud revenue of $28.5 billion, up 22% year-over-year;\n", + "•Microsoft Cloud revenue of $28.5 billion, up 25% year-over-year;\n", + "•Microsoft Cloud revenue of $28.5 billion, up 22% year-over-year;\n", + "•Microsoft Cloud revenue of $28.5 billion, up 25% year-over-year.\n", + "Gold Answer: Focused execution by our sales teams and partners\n", + "LLM Usage: {'input': 161, 'output': 87, 'total': 248, 'metric': 'tokens', 'processing_time': 40.02291202545166}\n", + "\n", + "16. Query: Which company is located in Nevada?\n", + "LLM Response: Red River Industries\n", + "Gold Answer: North Industries\n", + "LLM Usage: {'input': 96, 'output': 4, 'total': 100, 'metric': 'tokens', 'processing_time': 5.479044198989868}\n", + "\n", + "17. Query: When can termination after a material breach occur?\n", + "LLM Response: Not Found.\n", + "Gold Answer: If the breach is not cured within 15 days of notice of the breach.\n", + "LLM Usage: {'input': 134, 'output': 4, 'total': 138, 'metric': 'tokens', 'processing_time': 6.258498668670654}\n", + "\n", + "18. Query: What is a headline summary in 10 words or less?\n", + "LLM Response: •Joe Biden's tenure as the 46th president of the United States began with his inauguration on January 20, 2021.\n", + "•Biden, a Democrat from Delaware who previously served as vice president under Barack Obama, took office following his victory in the 2020 presidential election over Republican incumbent president Donald Trump.\n", + "Gold Answer: Joe Biden is the 46th President of the United States.\n", + "LLM Usage: {'input': 90, 'output': 63, 'total': 153, 'metric': 'tokens', 'processing_time': 27.374656677246094}\n", + "\n", + "19. Query: Who are the two people that won elections in Georgia?\n", + "LLM Response: Vivek Murthy and Marcella Nunez-Smith\n", + "Gold Answer: Jon Ossoff and Raphael Warnock\n", + "LLM Usage: {'input': 316, 'output': 15, 'total': 331, 'metric': 'tokens', 'processing_time': 17.75758385658264}\n", + "\n", + "20. Query: What is the list of the top financial highlights for the quarter?\n", + "LLM Response: •Revenue was $52.9 billion and increased 7% (up 10% in constant currency)\n", + "•Operating income was $22.4 billion and increased 10% (up 15% in constant currency)\n", + "•Net income was $18.3 billion and increased 9% (up 14% in constant currency)\n", + "•Diluted earnings per share was $2.45 and increased 10% (up 14% in constant currency).\n", + "Gold Answer: •Revenue: $52.9 million, up 10% in constant currency;\n", + "•Operating income: $22.4 billion, up 15% in constant currency;\n", + "•Net income: $18.3 billion, up 14% in constant currency;\n", + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.\n", + "LLM Usage: {'input': 164, 'output': 95, 'total': 259, 'metric': 'tokens', 'processing_time': 45.496915340423584}\n", + "\n", + "21. Query: What is a list of the key points?\n", + "LLM Response: •Stocks rallied Friday after the release of stronger-than-expected U.S. jobs data and a major increase in Treasury yields.\n", + "•The Dow Jones Industrial Average gained 195.12 points, or 0.76%, to close at 31,419.58.\n", + "•The S&P 500 added 1.59% at 4,008.50.\n", + "•The tech-heavy Nasdaq Composite added 1.35% at 4,008.50.\n", + "•The U.S. economy added 438,000 jobs in August, the Labor Department said.\n", + "•The economy added 437,000 jobs in August, but wages rose less than expected last month.\n", + "•Stocks posted a stunning turnaround on Friday, after initially falling on the stronger-than-expected jobs report.\n", + "•At its session low, the Dow had fallen as much as 198 points; it surged by more than 500 points at the height\n", + "Gold Answer: •Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in Treasury yields;\n", + "•Dow Jones gained 195.12 points;\n", + "•S&P 500 added 1.59%;\n", + "•Nasdaq Composite rose 1.35%;\n", + "•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n", + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.\n", + "LLM Usage: {'input': 456, 'output': 200, 'total': 656, 'metric': 'tokens', 'processing_time': 98.55418920516968}\n", + "\n", + "Total processing time: 432.05234456062317 seconds\n" + ] + } + ], + "source": [ + "\"\"\" This is the main example script - it loads the question list, loads the model and executes the prompts. \"\"\"\n", + "\n", + "t0 = time.time()\n", + "\n", + "print(f\"\\n > Loading Model: {model_name}...\")\n", + "\n", + "prompter = Prompt().load_model(model_name)\n", + "\n", + "t1 = time.time()\n", + "print(f\"\\n > Model {model_name} load time: {t1-t0} seconds\")\n", + "\n", + "for i, entries in enumerate(test_list):\n", + " print(f\"\\n{i+1}. Query: {entries['query']}\")\n", + "\n", + " # run the prompt\n", + " output = prompter.prompt_main(entries[\"query\"],\n", + " context=entries[\"context\"],\n", + " prompt_name=\"default_with_context\",\n", + " temperature=0.30)\n", + "\n", + " # 'output' is a dictionary with two keys - 'llm_response' and 'usage'\n", + " # --'llm_response' is the output from the model\n", + " # --'usage' is a dictionary with the usage stats\n", + "\n", + " llm_response = output[\"llm_response\"].strip(\"\\n\")\n", + " print(f\"LLM Response: {llm_response}\")\n", + "\n", + " # note: the 'gold answer' is the answer we provided above in the hello_world question list\n", + " print(f\"Gold Answer: {entries['answer']}\")\n", + "\n", + " print(f\"LLM Usage: {output['usage']}\")\n", + "\n", + "t2 = time.time()\n", + "print(f\"\\nTotal processing time: {t2-t1} seconds\")" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "042b0f51652c4104b32c9aae84cc3815": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2095638235c04b0082d5975a6a173776", + "placeholder": "​", + "style": "IPY_MODEL_b00019ae3408427085d30ae245044562", + "value": " 4.11G/4.11G [01:12<00:00, 50.2MB/s]" + } + }, + "16960e9d85944511a4e43d03f1acec1b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2095638235c04b0082d5975a6a173776": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "24e0c0e9e25b417a8f6f35a952eba293": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "26ca04a2a9be4232bd5ba979d280ef8d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_6dd1fc3a33c5428580836442e77b8b8c", + "IPY_MODEL_99b9def487bc4cbd8723a34865d36487", + "IPY_MODEL_ce9e2fca4c334772a6fae7936454f530" + ], + "layout": "IPY_MODEL_c6e421b0df86447ea47a08132539e200" + } + }, + "2d5d40b7513348bba66e6bf57da39e60": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_858187f2ed144dc59ee5cd70d49a09c4", + "max": 2270, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_726d4b1cafad4c84898b9bbeb31bb21b", + "value": 2270 + } + }, + "31e2c815ea164d888c5276310fd8ddfb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "41a1a662e00a494296c18901026f75ae": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "4230b717baef43eaa0c4a7ca4aad207f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_74d20e36afe740839b19b913404bb617", + "placeholder": "​", + "style": "IPY_MODEL_ababdb41145e4524babd99a4d061f375", + "value": "pytorch_model.bin: 100%" + } + }, + "6dd1fc3a33c5428580836442e77b8b8c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c1b3b62618084dc4a9c3451a081cc3d8", + "placeholder": "​", + "style": "IPY_MODEL_e7abfcc177204428b5891ddfe64877db", + "value": "tokenizer.json: 100%" + } + }, + "726d4b1cafad4c84898b9bbeb31bb21b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "74d20e36afe740839b19b913404bb617": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "81f9f227f2aa47bfb007f2a7c1598bfa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "82fedb4609f84ed09b5f0e578b73875c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "858187f2ed144dc59ee5cd70d49a09c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "952644e2c51845c49b5093498c087c33": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "96687e6808d54377a8bb8d5a0cb9e6ff": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_81f9f227f2aa47bfb007f2a7c1598bfa", + "placeholder": "​", + "style": "IPY_MODEL_31e2c815ea164d888c5276310fd8ddfb", + "value": "config.json: 100%" + } + }, + "99b9def487bc4cbd8723a34865d36487": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ff04ff6ce4f24e2f956fa8870079b012", + "max": 2113837, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_41a1a662e00a494296c18901026f75ae", + "value": 2113837 + } + }, + "99bad8042d254c6792d40a3444db1ce9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_952644e2c51845c49b5093498c087c33", + "placeholder": "​", + "style": "IPY_MODEL_82fedb4609f84ed09b5f0e578b73875c", + "value": " 2.27k/2.27k [00:00<00:00, 81.5kB/s]" + } + }, + "9afd954348c342d1ac047cc78f544bbe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9c8fafd21215457390a63c5ff6a36a50": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9afd954348c342d1ac047cc78f544bbe", + "max": 4114329821, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_b0b170d1214541b18ee333ac1709061b", + "value": 4114329821 + } + }, + "ababdb41145e4524babd99a4d061f375": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b00019ae3408427085d30ae245044562": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b0b170d1214541b18ee333ac1709061b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "ba44a68fc00944dd844cac4ea9c2f507": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c1b3b62618084dc4a9c3451a081cc3d8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c6e421b0df86447ea47a08132539e200": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cc779a09790d4186a6832dae389af0bf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ce9e2fca4c334772a6fae7936454f530": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_16960e9d85944511a4e43d03f1acec1b", + "placeholder": "​", + "style": "IPY_MODEL_cc779a09790d4186a6832dae389af0bf", + "value": " 2.11M/2.11M [00:00<00:00, 3.82MB/s]" + } + }, + "e46164dbf38e4f929942405eee41746b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_96687e6808d54377a8bb8d5a0cb9e6ff", + "IPY_MODEL_2d5d40b7513348bba66e6bf57da39e60", + "IPY_MODEL_99bad8042d254c6792d40a3444db1ce9" + ], + "layout": "IPY_MODEL_ba44a68fc00944dd844cac4ea9c2f507" + } + }, + "e7abfcc177204428b5891ddfe64877db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ea33d6532d7f4ad8af85c40588162e5f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4230b717baef43eaa0c4a7ca4aad207f", + "IPY_MODEL_9c8fafd21215457390a63c5ff6a36a50", + "IPY_MODEL_042b0f51652c4104b32c9aae84cc3815" + ], + "layout": "IPY_MODEL_24e0c0e9e25b417a8f6f35a952eba293" + } + }, + "ff04ff6ce4f24e2f956fa8870079b012": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_4_rag_text_query.ipynb b/tutorials/notebooks/fast_start_examples/example_4_rag_text_query.ipynb new file mode 100644 index 0000000..4a39d73 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_4_rag_text_query.ipynb @@ -0,0 +1,1628 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "62BSwEddSet9", + "outputId": "d8a34e1b-9ce1-48d1-c138-42c790139eff" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.2.14-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3==1.24.53 (from llmware)\n", + " Downloading boto3-1.24.53-py3-none-any.whl (132 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.5/132.5 kB\u001b[0m \u001b[31m795.1 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.0)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting openai>=1.0.0 (from llmware)\n", + " Downloading openai-1.30.1-py3-none-any.whl (320 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (670 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.0/670.0 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from llmware) (2.3.0+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.41.0)\n", + "Collecting Wikipedia-API==0.6.0 (from llmware)\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m40.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Collecting einops==0.7.0 (from llmware)\n", + " Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Collecting botocore<1.28.0,>=1.27.53 (from boto3==1.24.53->llmware)\n", + " Downloading botocore-1.27.96-py3-none-any.whl (9.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m52.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3==1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.7.0,>=0.6.0 (from boto3==1.24.53->llmware)\n", + " Downloading s3transfer-0.6.2-py3-none-any.whl (79 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.11.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Wikipedia-API==0.6.0->llmware) (2.31.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->llmware) (1.7.0)\n", + "Collecting httpx<1,>=0.23.0 (from openai>=1.0.0->llmware)\n", + " Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (2.7.1)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (1.3.1)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (3.1.4)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n", + "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (2.3.0)\n", + "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->llmware)\n", + " Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m39.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware) (0.4.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (3.7)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (1.2.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.8.2)\n", + "Collecting urllib3<1.27,>=1.25.4 (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware)\n", + " Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2024.2.2)\n", + "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.0.0->llmware)\n", + " Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware)\n", + " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->llmware) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: urllib3, psycopg-binary, psycopg, pgvector, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, jmespath, h11, einops, dnspython, colorama, pymongo, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpcore, botocore, Wikipedia-API, s3transfer, nvidia-cusolver-cu12, httpx, openai, boto3, llmware\n", + " Attempting uninstall: urllib3\n", + " Found existing installation: urllib3 2.0.7\n", + " Uninstalling urllib3-2.0.7:\n", + " Successfully uninstalled urllib3-2.0.7\n", + "Successfully installed Wikipedia-API-0.6.0 boto3-1.24.53 botocore-1.27.96 colorama-0.4.6 dnspython-2.6.1 einops-0.7.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 jmespath-1.0.1 llmware-0.2.14 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 openai-1.30.1 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.2 s3transfer-0.6.2 urllib3-1.26.18\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "nWRAUgFFSnpQ" + }, + "outputs": [], + "source": [ + "import os\n", + "import re\n", + "from llmware.prompts import Prompt, HumanInTheLoop\n", + "from llmware.setup import Setup\n", + "from llmware.configs import LLMWareConfig\n", + "from llmware.retrieval import Query\n", + "from llmware.library import Library" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "rqCmLej_Skmb" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "example_models = [\"llmware/bling-1b-0.1\", \"llmware/bling-tiny-llama-v0\", \"llmware/dragon-yi-6b-gguf\"]\n", + "model_name = example_models[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0QRIU4YuSurX", + "outputId": "e74db71d-fa1f-4fe0-c060-7085fd56c07a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " > Loading the llmware sample files...\n" + ] + }, + { + "data": { + "text/plain": [ + "{'docs_added': 15,\n", + " 'blocks_added': 2211,\n", + " 'images_added': 0,\n", + " 'pages_added': 204,\n", + " 'tables_added': 0,\n", + " 'rejected_files': []}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\"\"\" Example #4a: Main general case to run a RAG workflow from a Library \"\"\"\n", + "\n", + "# Load the llmware sample files\n", + "print (f\"\\n > Loading the llmware sample files...\")\n", + "sample_files_path = Setup().load_sample_files()\n", + "contracts_path = os.path.join(sample_files_path,\"Agreements\")\n", + "\n", + "contracts_lib = Library().create_new_library(\"example4_library\")\n", + "contracts_lib.add_files(contracts_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MjftxdsC1wEM", + "outputId": "5bdc7c62-3a14-4d06-9d63-44fb79b3874a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " > Loading model llmware/bling-1b-0.1...\n" + ] + } + ], + "source": [ + "# questions that we want to ask each contract\n", + "question_list = [{\"topic\": \"executive employment agreement\", \"llm_query\": \"What are the names of the two parties?\"},\n", + " {\"topic\": \"base salary\", \"llm_query\": \"What is the executive's base salary?\"},\n", + " {\"topic\": \"governing law\", \"llm_query\": \"What is the governing law?\"}]\n", + "\n", + "print (f\"\\n > Loading model {model_name}...\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "NOD3tzMT14Pj" + }, + "outputs": [], + "source": [ + "# get a list of all of the unique documents in the library\n", + "q = Query(contracts_lib)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "MBo9BF292CeD", + "outputId": "0ef55a42-ee11-42bf-e2e9-dd993a90654f" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: document id list - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]\n" + ] + } + ], + "source": [ + "# doc id list\n", + "doc_list = q.list_doc_id()\n", + "print(\"update: document id list - \", doc_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "szxbQbsw2EuR", + "outputId": "896dd819-3462-419d-a456-8556dd1d0caa" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: filename list - ['Artemis Poseidon EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Leto EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Bia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Amphitrite EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Rhea EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Gaia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Nike EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Nyx EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Demeter EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Persephone EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Metis EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Apollo EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Eileithyia EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Athena EXECUTIVE EMPLOYMENT AGREEMENT.pdf', 'Aphrodite EXECUTIVE EMPLOYMENT AGREEMENT.pdf']\n" + ] + } + ], + "source": [ + "# filename list\n", + "fn_list = q.list_doc_fn()\n", + "print(\"update: filename list - \", fn_list)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "9baa1a8be386404bbe93e48fe0258772", + "7a4dfb45d16242459cf955f6b584c65a", + "1b287a7ff2cd4c29b75f847e8f47d504", + "b90effaf886b488d92ce9ac317295e33", + "a159af95aba94188a628d41af4ef4e62", + "9b9de417e6b74746bbaf9a6f20e9aa05", + "5950d7a9d44b447895ca42bfe88c01a9", + "c1b76d81f3f943c6b561965a0ba05e81", + "345b8447f21b4236a6480fe7b9407441", + "440e0638a0494dbd83da8270fb4ca2e6", + "6948bef17922418cbe7a49dd67a81528", + "dc7dfd699a5b43f19810414a46388e84", + "828243d927ef4accb42f5cd77c066ad0", + "8ebc938a2e454fb5b0d66a56995ba994", + "9923f776799347698efc1003448f8934", + "c2356643c5514b7ead063ec4249ac720", + "02f7615f5d6e45428406acb435d17a2d", + "527f87914ae54088a98f41aa1e3f4405", + "cfbc13c05e1a4fa7ae3734de099983f0", + "305c036506bf4fe0ad16215ebdae985b", + "4cf48f1a172844d09b7be62b74756093", + "886e8fda6aac4182a64e4900f57d391b", + "8578fefe65d048bfbc28b151e4473e1e", + "cde999cb89394b86ac665f6d75977552", + "4625d486912d4305870312c392da045c", + "8e905b36221146da9ae642eb66dba2d0", + "407dccba55e8491aa0b38a2540e3f5b7", + "38794db38afc44aea4c1329647ac78fe", + "7813c622590b4c29a1dc5138681980f7", + "6d2e312415c14745837b0f411cefdb58", + "3f4b6a8bcc3f41988a5c6e69b96b31ca", + "4503a8be136f4b1791fa38db39130eae", + "ac30042a94c64903b5e61ade8d8a9dcb" + ] + }, + "id": "ElWoQX1x2K9E", + "outputId": "c261a794-2f5e-4ffc-9e4a-0a3e443635cd" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "9baa1a8be386404bbe93e48fe0258772", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "config.json: 0%| | 0.00/2.27k [00:00" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Save jsonl report to jsonl to /prompt_history folder\n", + "print(\"\\nPrompt state saved at: \", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id))\n", + "prompter.save_state()" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "75cPnxBw2Rax", + "outputId": "87f7b00d-e5d1-4cc1-e4ab-9a5e4b452b43" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "CSV output saved at: {'report_name': 'interaction_report_Wed May 22 13:02:20 2024.csv', 'report_fp': '/root/llmware_data/prompt_history/interaction_report_Wed May 22 13:02:20 2024.csv', 'results': 0}\n" + ] + } + ], + "source": [ + "# Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review\n", + "csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv()\n", + "print(\"\\nCSV output saved at: \", csv_output)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "02f7615f5d6e45428406acb435d17a2d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1b287a7ff2cd4c29b75f847e8f47d504": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c1b76d81f3f943c6b561965a0ba05e81", + "max": 2270, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_345b8447f21b4236a6480fe7b9407441", + "value": 2270 + } + }, + "305c036506bf4fe0ad16215ebdae985b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "345b8447f21b4236a6480fe7b9407441": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "38794db38afc44aea4c1329647ac78fe": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3f4b6a8bcc3f41988a5c6e69b96b31ca": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "407dccba55e8491aa0b38a2540e3f5b7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "440e0638a0494dbd83da8270fb4ca2e6": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4503a8be136f4b1791fa38db39130eae": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4625d486912d4305870312c392da045c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6d2e312415c14745837b0f411cefdb58", + "max": 2113837, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3f4b6a8bcc3f41988a5c6e69b96b31ca", + "value": 2113837 + } + }, + "4cf48f1a172844d09b7be62b74756093": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "527f87914ae54088a98f41aa1e3f4405": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5950d7a9d44b447895ca42bfe88c01a9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6948bef17922418cbe7a49dd67a81528": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6d2e312415c14745837b0f411cefdb58": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7813c622590b4c29a1dc5138681980f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7a4dfb45d16242459cf955f6b584c65a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9b9de417e6b74746bbaf9a6f20e9aa05", + "placeholder": "​", + "style": "IPY_MODEL_5950d7a9d44b447895ca42bfe88c01a9", + "value": "config.json: 100%" + } + }, + "828243d927ef4accb42f5cd77c066ad0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_02f7615f5d6e45428406acb435d17a2d", + "placeholder": "​", + "style": "IPY_MODEL_527f87914ae54088a98f41aa1e3f4405", + "value": "pytorch_model.bin: 100%" + } + }, + "8578fefe65d048bfbc28b151e4473e1e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_cde999cb89394b86ac665f6d75977552", + "IPY_MODEL_4625d486912d4305870312c392da045c", + "IPY_MODEL_8e905b36221146da9ae642eb66dba2d0" + ], + "layout": "IPY_MODEL_407dccba55e8491aa0b38a2540e3f5b7" + } + }, + "886e8fda6aac4182a64e4900f57d391b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "8e905b36221146da9ae642eb66dba2d0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4503a8be136f4b1791fa38db39130eae", + "placeholder": "​", + "style": "IPY_MODEL_ac30042a94c64903b5e61ade8d8a9dcb", + "value": " 2.11M/2.11M [00:00<00:00, 2.36MB/s]" + } + }, + "8ebc938a2e454fb5b0d66a56995ba994": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cfbc13c05e1a4fa7ae3734de099983f0", + "max": 4114329821, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_305c036506bf4fe0ad16215ebdae985b", + "value": 4114329821 + } + }, + "9923f776799347698efc1003448f8934": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4cf48f1a172844d09b7be62b74756093", + "placeholder": "​", + "style": "IPY_MODEL_886e8fda6aac4182a64e4900f57d391b", + "value": " 4.11G/4.11G [01:02<00:00, 75.3MB/s]" + } + }, + "9b9de417e6b74746bbaf9a6f20e9aa05": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9baa1a8be386404bbe93e48fe0258772": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7a4dfb45d16242459cf955f6b584c65a", + "IPY_MODEL_1b287a7ff2cd4c29b75f847e8f47d504", + "IPY_MODEL_b90effaf886b488d92ce9ac317295e33" + ], + "layout": "IPY_MODEL_a159af95aba94188a628d41af4ef4e62" + } + }, + "a159af95aba94188a628d41af4ef4e62": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ac30042a94c64903b5e61ade8d8a9dcb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "b90effaf886b488d92ce9ac317295e33": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_440e0638a0494dbd83da8270fb4ca2e6", + "placeholder": "​", + "style": "IPY_MODEL_6948bef17922418cbe7a49dd67a81528", + "value": " 2.27k/2.27k [00:00<00:00, 92.0kB/s]" + } + }, + "c1b76d81f3f943c6b561965a0ba05e81": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c2356643c5514b7ead063ec4249ac720": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "cde999cb89394b86ac665f6d75977552": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_38794db38afc44aea4c1329647ac78fe", + "placeholder": "​", + "style": "IPY_MODEL_7813c622590b4c29a1dc5138681980f7", + "value": "tokenizer.json: 100%" + } + }, + "cfbc13c05e1a4fa7ae3734de099983f0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "dc7dfd699a5b43f19810414a46388e84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_828243d927ef4accb42f5cd77c066ad0", + "IPY_MODEL_8ebc938a2e454fb5b0d66a56995ba994", + "IPY_MODEL_9923f776799347698efc1003448f8934" + ], + "layout": "IPY_MODEL_c2356643c5514b7ead063ec4249ac720" + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_5_rag_semantic_query.ipynb b/tutorials/notebooks/fast_start_examples/example_5_rag_semantic_query.ipynb new file mode 100644 index 0000000..c3f370e --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_5_rag_semantic_query.ipynb @@ -0,0 +1,614 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0UYukB3iTsl8", + "outputId": "c349700e-6a00-452a-f5aa-e04183637c5d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.2.14-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3==1.24.53 (from llmware)\n", + " Downloading boto3-1.24.53-py3-none-any.whl (132 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.5/132.5 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.0)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting openai>=1.0.0 (from llmware)\n", + " Downloading openai-1.30.1-py3-none-any.whl (320 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m23.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (670 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.0/670.0 kB\u001b[0m \u001b[31m17.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from llmware) (2.3.0+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.41.0)\n", + "Collecting Wikipedia-API==0.6.0 (from llmware)\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m48.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Collecting einops==0.7.0 (from llmware)\n", + " Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Collecting botocore<1.28.0,>=1.27.53 (from boto3==1.24.53->llmware)\n", + " Downloading botocore-1.27.96-py3-none-any.whl (9.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m38.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3==1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.7.0,>=0.6.0 (from boto3==1.24.53->llmware)\n", + " Downloading s3transfer-0.6.2-py3-none-any.whl (79 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.11.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Wikipedia-API==0.6.0->llmware) (2.31.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.0)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->llmware) (1.7.0)\n", + "Collecting httpx<1,>=0.23.0 (from openai>=1.0.0->llmware)\n", + " Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (2.7.1)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware) (1.3.1)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (3.1.4)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n", + "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.13.1->llmware)\n", + " Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware) (2.3.0)\n", + "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->llmware)\n", + " Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m62.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware) (0.4.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (3.7)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware) (1.2.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (2.8.2)\n", + "Collecting urllib3<1.27,>=1.25.4 (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware)\n", + " Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m14.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware) (2024.2.2)\n", + "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.0.0->llmware)\n", + " Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware)\n", + " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Wikipedia-API==0.6.0->llmware) (3.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->llmware) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->llmware) (1.3.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: urllib3, psycopg-binary, psycopg, pgvector, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, jmespath, h11, einops, dnspython, colorama, pymongo, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpcore, botocore, Wikipedia-API, s3transfer, nvidia-cusolver-cu12, httpx, openai, boto3, llmware\n", + " Attempting uninstall: urllib3\n", + " Found existing installation: urllib3 2.0.7\n", + " Uninstalling urllib3-2.0.7:\n", + " Successfully uninstalled urllib3-2.0.7\n", + "Successfully installed Wikipedia-API-0.6.0 boto3-1.24.53 botocore-1.27.96 colorama-0.4.6 dnspython-2.6.1 einops-0.7.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 jmespath-1.0.1 llmware-0.2.14 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 openai-1.30.1 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.2 s3transfer-0.6.2 urllib3-1.26.18\n", + "Collecting faiss-cpu\n", + " Downloading faiss_cpu-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (27.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.0/27.0 MB\u001b[0m \u001b[31m47.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from faiss-cpu) (1.25.2)\n", + "Installing collected packages: faiss-cpu\n", + "Successfully installed faiss-cpu-1.8.0\n", + "Collecting chromadb\n", + " Downloading chromadb-0.5.0-py3-none-any.whl (526 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m526.8/526.8 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: build>=1.0.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.2.1)\n", + "Requirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.31.0)\n", + "Requirement already satisfied: pydantic>=1.9 in /usr/local/lib/python3.10/dist-packages (from chromadb) (2.7.1)\n", + "Collecting chroma-hnswlib==0.7.3 (from chromadb)\n", + " Downloading chroma_hnswlib-0.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m53.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting fastapi>=0.95.2 (from chromadb)\n", + " Downloading fastapi-0.111.0-py3-none-any.whl (91 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.0/92.0 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting uvicorn[standard]>=0.18.3 (from chromadb)\n", + " Downloading uvicorn-0.29.0-py3-none-any.whl (60 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.22.5 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.25.2)\n", + "Collecting posthog>=2.4.0 (from chromadb)\n", + " Downloading posthog-3.5.0-py2.py3-none-any.whl (41 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.3/41.3 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=4.5.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.11.0)\n", + "Collecting onnxruntime>=1.14.1 (from chromadb)\n", + " Downloading onnxruntime-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl (6.8 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m80.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-api>=1.2.0 (from chromadb)\n", + " Downloading opentelemetry_api-1.24.0-py3-none-any.whl (60 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.1/60.1 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-exporter-otlp-proto-grpc>=1.2.0 (from chromadb)\n", + " Downloading opentelemetry_exporter_otlp_proto_grpc-1.24.0-py3-none-any.whl (18 kB)\n", + "Collecting opentelemetry-instrumentation-fastapi>=0.41b0 (from chromadb)\n", + " Downloading opentelemetry_instrumentation_fastapi-0.45b0-py3-none-any.whl (11 kB)\n", + "Collecting opentelemetry-sdk>=1.2.0 (from chromadb)\n", + " Downloading opentelemetry_sdk-1.24.0-py3-none-any.whl (106 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.1/106.1 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.13.2 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.19.1)\n", + "Collecting pypika>=0.48.9 (from chromadb)\n", + " Downloading PyPika-0.48.9.tar.gz (67 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.3/67.3 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: tqdm>=4.65.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (4.66.4)\n", + "Collecting overrides>=7.3.1 (from chromadb)\n", + " Downloading overrides-7.7.0-py3-none-any.whl (17 kB)\n", + "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.4.0)\n", + "Requirement already satisfied: grpcio>=1.58.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (1.63.0)\n", + "Collecting bcrypt>=4.0.1 (from chromadb)\n", + " Downloading bcrypt-4.1.3-cp39-abi3-manylinux_2_28_x86_64.whl (283 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m283.7/283.7 kB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typer>=0.9.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (0.9.4)\n", + "Collecting kubernetes>=28.1.0 (from chromadb)\n", + " Downloading kubernetes-29.0.0-py2.py3-none-any.whl (1.6 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m63.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tenacity>=8.2.3 in /usr/local/lib/python3.10/dist-packages (from chromadb) (8.3.0)\n", + "Requirement already satisfied: PyYAML>=6.0.0 in /usr/local/lib/python3.10/dist-packages (from chromadb) (6.0.1)\n", + "Collecting mmh3>=4.0.1 (from chromadb)\n", + " Downloading mmh3-4.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (67 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.6/67.6 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting orjson>=3.9.12 (from chromadb)\n", + " Downloading orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (142 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.5/142.5 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: packaging>=19.1 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (24.0)\n", + "Requirement already satisfied: pyproject_hooks in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (1.1.0)\n", + "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from build>=1.0.3->chromadb) (2.0.1)\n", + "Collecting starlette<0.38.0,>=0.37.2 (from fastapi>=0.95.2->chromadb)\n", + " Downloading starlette-0.37.2-py3-none-any.whl (71 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.9/71.9 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting fastapi-cli>=0.0.2 (from fastapi>=0.95.2->chromadb)\n", + " Downloading fastapi_cli-0.0.4-py3-none-any.whl (9.5 kB)\n", + "Requirement already satisfied: httpx>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (0.27.0)\n", + "Requirement already satisfied: jinja2>=2.11.2 in /usr/local/lib/python3.10/dist-packages (from fastapi>=0.95.2->chromadb) (3.1.4)\n", + "Collecting python-multipart>=0.0.7 (from fastapi>=0.95.2->chromadb)\n", + " Downloading python_multipart-0.0.9-py3-none-any.whl (22 kB)\n", + "Collecting ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 (from fastapi>=0.95.2->chromadb)\n", + " Downloading ujson-5.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (53 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting email_validator>=2.0.0 (from fastapi>=0.95.2->chromadb)\n", + " Downloading email_validator-2.1.1-py3-none-any.whl (30 kB)\n", + "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2024.2.2)\n", + "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n", + "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (2.27.0)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n", + "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n", + "Requirement already satisfied: oauthlib>=3.2.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n", + "Requirement already satisfied: urllib3>=1.24.2 in /usr/local/lib/python3.10/dist-packages (from kubernetes>=28.1.0->chromadb) (1.26.18)\n", + "Collecting coloredlogs (from onnxruntime>=1.14.1->chromadb)\n", + " Downloading coloredlogs-15.0.1-py2.py3-none-any.whl (46 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: flatbuffers in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\n", + "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (3.20.3)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n", + "Collecting deprecated>=1.2.6 (from opentelemetry-api>=1.2.0->chromadb)\n", + " Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting importlib-metadata<=7.0,>=6.0 (from opentelemetry-api>=1.2.0->chromadb)\n", + " Downloading importlib_metadata-7.0.0-py3-none-any.whl (23 kB)\n", + "Requirement already satisfied: googleapis-common-protos~=1.52 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.63.0)\n", + "Collecting opentelemetry-exporter-otlp-proto-common==1.24.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n", + " Downloading opentelemetry_exporter_otlp_proto_common-1.24.0-py3-none-any.whl (17 kB)\n", + "Collecting opentelemetry-proto==1.24.0 (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb)\n", + " Downloading opentelemetry_proto-1.24.0-py3-none-any.whl (50 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.8/50.8 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting opentelemetry-instrumentation-asgi==0.45b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_instrumentation_asgi-0.45b0-py3-none-any.whl (14 kB)\n", + "Collecting opentelemetry-instrumentation==0.45b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_instrumentation-0.45b0-py3-none-any.whl (28 kB)\n", + "Collecting opentelemetry-semantic-conventions==0.45b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_semantic_conventions-0.45b0-py3-none-any.whl (36 kB)\n", + "Collecting opentelemetry-util-http==0.45b0 (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading opentelemetry_util_http-0.45b0-py3-none-any.whl (6.9 kB)\n", + "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.45b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (67.7.2)\n", + "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from opentelemetry-instrumentation==0.45b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.14.1)\n", + "Collecting asgiref~=3.0 (from opentelemetry-instrumentation-asgi==0.45b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb)\n", + " Downloading asgiref-3.8.1-py3-none-any.whl (23 kB)\n", + "Collecting monotonic>=1.5 (from posthog>=2.4.0->chromadb)\n", + " Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n", + "Collecting backoff>=1.10.0 (from posthog>=2.4.0->chromadb)\n", + " Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic>=1.9->chromadb) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.28->chromadb) (3.7)\n", + "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from tokenizers>=0.13.2->chromadb) (0.23.0)\n", + "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (8.1.7)\n", + "Requirement already satisfied: h11>=0.8 in /usr/local/lib/python3.10/dist-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n", + "Collecting httptools>=0.5.0 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (341 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m341.4/341.4 kB\u001b[0m \u001b[31m33.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting python-dotenv>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n", + "Collecting uvloop!=0.15.0,!=0.15.1,>=0.14.0 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m90.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting watchfiles>=0.13 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m52.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting websockets>=10.4 (from uvicorn[standard]>=0.18.3->chromadb)\n", + " Downloading websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (130 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m130.2/130.2 kB\u001b[0m \u001b[31m15.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: dnspython>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from email_validator>=2.0.0->fastapi>=0.95.2->chromadb) (2.6.1)\n", + "Collecting typer>=0.9.0 (from chromadb)\n", + " Downloading typer-0.12.3-py3-none-any.whl (47 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.2/47.2 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting shellingham>=1.3.0 (from typer>=0.9.0->chromadb)\n", + " Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n", + "Requirement already satisfied: rich>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer>=0.9.0->chromadb) (13.7.1)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.3)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n", + "Requirement already satisfied: anyio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi>=0.95.2->chromadb) (3.7.1)\n", + "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi>=0.95.2->chromadb) (1.0.5)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx>=0.23.0->fastapi>=0.95.2->chromadb) (1.3.1)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.6.0)\n", + "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/dist-packages (from importlib-metadata<=7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.18.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2>=2.11.2->fastapi>=0.95.2->chromadb) (2.1.5)\n", + "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (3.0.0)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (2.16.1)\n", + "Collecting humanfriendly>=9.1 (from coloredlogs->onnxruntime>=1.14.1->chromadb)\n", + " Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio->httpx>=0.23.0->fastapi>=0.95.2->chromadb) (1.2.1)\n", + "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb) (0.1.2)\n", + "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.0)\n", + "Building wheels for collected packages: pypika\n", + " Building wheel for pypika (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for pypika: filename=PyPika-0.48.9-py2.py3-none-any.whl size=53724 sha256=297657f4f0a77da36d766967d5961b20ee3f55c8b33f862efcd4e084ce7a31d2\n", + " Stored in directory: /root/.cache/pip/wheels/e1/26/51/d0bffb3d2fd82256676d7ad3003faea3bd6dddc9577af665f4\n", + "Successfully built pypika\n", + "Installing collected packages: pypika, monotonic, mmh3, websockets, uvloop, uvicorn, ujson, shellingham, python-multipart, python-dotenv, overrides, orjson, opentelemetry-util-http, opentelemetry-semantic-conventions, opentelemetry-proto, importlib-metadata, humanfriendly, httptools, email_validator, deprecated, chroma-hnswlib, bcrypt, backoff, asgiref, watchfiles, starlette, posthog, opentelemetry-exporter-otlp-proto-common, opentelemetry-api, coloredlogs, typer, opentelemetry-sdk, opentelemetry-instrumentation, onnxruntime, kubernetes, opentelemetry-instrumentation-asgi, opentelemetry-exporter-otlp-proto-grpc, fastapi-cli, opentelemetry-instrumentation-fastapi, fastapi, chromadb\n", + " Attempting uninstall: importlib-metadata\n", + " Found existing installation: importlib_metadata 7.1.0\n", + " Uninstalling importlib_metadata-7.1.0:\n", + " Successfully uninstalled importlib_metadata-7.1.0\n", + " Attempting uninstall: typer\n", + " Found existing installation: typer 0.9.4\n", + " Uninstalling typer-0.9.4:\n", + " Successfully uninstalled typer-0.9.4\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "spacy 3.7.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\n", + "weasel 0.3.4 requires typer<0.10.0,>=0.3.0, but you have typer 0.12.3 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed asgiref-3.8.1 backoff-2.2.1 bcrypt-4.1.3 chroma-hnswlib-0.7.3 chromadb-0.5.0 coloredlogs-15.0.1 deprecated-1.2.14 email_validator-2.1.1 fastapi-0.111.0 fastapi-cli-0.0.4 httptools-0.6.1 humanfriendly-10.0 importlib-metadata-7.0.0 kubernetes-29.0.0 mmh3-4.1.0 monotonic-1.6 onnxruntime-1.18.0 opentelemetry-api-1.24.0 opentelemetry-exporter-otlp-proto-common-1.24.0 opentelemetry-exporter-otlp-proto-grpc-1.24.0 opentelemetry-instrumentation-0.45b0 opentelemetry-instrumentation-asgi-0.45b0 opentelemetry-instrumentation-fastapi-0.45b0 opentelemetry-proto-1.24.0 opentelemetry-sdk-1.24.0 opentelemetry-semantic-conventions-0.45b0 opentelemetry-util-http-0.45b0 orjson-3.10.3 overrides-7.7.0 posthog-3.5.0 pypika-0.48.9 python-dotenv-1.0.1 python-multipart-0.0.9 shellingham-1.5.4 starlette-0.37.2 typer-0.12.3 ujson-5.10.0 uvicorn-0.29.0 uvloop-0.19.0 watchfiles-0.21.0 websockets-12.0\n" + ] + } + ], + "source": [ + "!git install llmware\n", + "!pip install faiss-cpu\n", + "!pip install chromadb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sFO2Ds1RUfwe" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.library import Library\n", + "from llmware.retrieval import Query\n", + "from llmware.setup import Setup\n", + "from llmware.resources import Status\n", + "from llmware.prompts import Prompt\n", + "from llmware.configs import LLMWareConfig\n", + "from importlib import util\n", + "if not util.find_spec(\"chromadb\"):\n", + " print(\"\\nto run this example with chromadb python sdk: pip3 install chromadb\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uhrlGycBUrCG" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "2VBaVb_PVWOJ" + }, + "outputs": [], + "source": [ + "embedding_model = \"industry-bert-contracts\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "6a1XOi2_Xtd3" + }, + "outputs": [], + "source": [ + "vector_db = \"chomadb\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "wi8MwN4PYUSb" + }, + "outputs": [], + "source": [ + "lib_name = \"example_5_library\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "uI4HDK7aakoT" + }, + "outputs": [], + "source": [ + "example_models = [\"llmware/bling-1b-0.1\", \"llmware/bling-tiny-llama-v0\", \"llmware/dragon-yi-6b-gguf\"]\n", + "llm_model_name = example_models[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RPSiEvNda1Kn", + "outputId": "86c30538-735a-4f65-f90d-f35d13bffe4a" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: Step 1 - Creating library: example_5_library\n" + ] + } + ], + "source": [ + "print(\"\\nupdate: Step 1 - Creating library: {}\".format(lib_name))\n", + "library = Library().create_new_library(lib_name)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "dnC1axdCc_-m", + "outputId": "6bfa4593-e3a1-412d-8833-3444cad6f9bd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Step 2 - Downloading Sample Files\n" + ] + } + ], + "source": [ + "print(\"update: Step 2 - Downloading Sample Files\")\n", + "sample_files_path = Setup().load_sample_files(over_write=False)\n", + "contracts_path = os.path.join(sample_files_path, \"Agreements\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4g2zsJgsdJ20", + "outputId": "d41daad4-e372-4d00-f76b-d502cf9c9dd5" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Step 3 - Parsing and Text Indexing Files\n" + ] + }, + { + "data": { + "text/plain": [ + "{'docs_added': 0,\n", + " 'blocks_added': 0,\n", + " 'images_added': 0,\n", + " 'pages_added': 0,\n", + " 'tables_added': 0,\n", + " 'rejected_files': []}" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(\"update: Step 3 - Parsing and Text Indexing Files\")\n", + "library.add_files(input_folder_path=contracts_path, chunk_size=400, max_chunk_size=600, smart_chunking=1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 211 + }, + "id": "jsIJAU8ddq7X", + "outputId": "ce048992-d42d-4bf2-f7bf-67ee26ad45c9" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "update: Step 4 - Generating Embeddings in chomadb db - with Model- industry-bert-contracts\n" + ] + }, + { + "ename": "TypeError", + "evalue": "Library.install_new_embedding() got an unexpected keyword argument 'embedding_model'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvector_db\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_model\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mlibrary\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minstall_new_embedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membedding_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0membedding_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvector_db\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvector_db\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;31mTypeError\u001b[0m: Library.install_new_embedding() got an unexpected keyword argument 'embedding_model'" + ] + } + ], + "source": [ + "print(\"\\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}\".format(vector_db, embedding_model))\n", + "library.install_new_embedding(embedding_model=embedding_model, vector_db=vector_db)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "y9fcyXRr1EFQ" + }, + "outputs": [], + "source": [ + "print(\"\\nupdate: Loading model for LLM inference - \", llm_model_name)\n", + "prompter = Prompt().load_model(llm_model_name)\n", + "query = \"what is the executive's base annual salary\"\n", + "results = Query(library).semantic_query(query, result_count=50, embedding_distance_threshold=1.0)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NabMd1yU1wdU" + }, + "outputs": [], + "source": [ + "for i, contract in enumerate(os.listdir(contracts_path_)):\n", + "\n", + " qr = []\n", + "\n", + " if contract != \".DS_Store\":\n", + " print(\"\\nContract Name: \", i, contract)\n", + "\n", + " for j, entries in enumerate(results):\n", + " library_fn = entries[\"file_source\"]\n", + "\n", + " if os.sep in library_fn:\n", + " library_fn = library_fn.split(os.sep)[-1]\n", + "\n", + " if library_fn == contract:\n", + " print(\"Top Retrieval: \", j, entries[\"distance\"], entries[\"text\"])\n", + " qr.append(entries)\n", + "\n", + " source = prompter.add_source_query_results(query_results=qr)\n", + "\n", + " respose = prompter.prompt_with_source(query, prompt_name=\"default_with_context\", temperature=0.3)\n", + "\n", + " for resp in response:\n", + " if \"llm_response\" in resp:\n", + " print(\"\\nupdate: llm answer - \", resp[\"llm_response\"])\n", + "\n", + " prompter.clear_source_materials()\n", + "\n", + " return 0" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_6_rag_multi_step_query_version_1ipynb.ipynb b/tutorials/notebooks/fast_start_examples/example_6_rag_multi_step_query_version_1ipynb.ipynb new file mode 100644 index 0000000..63a23d5 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_6_rag_multi_step_query_version_1ipynb.ipynb @@ -0,0 +1,267 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "1nQ_rskY0eBd" + }, + "source": [ + "# **If you are using Colab for free, we highly recommend you active the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed to the 13GB Colab gives automatically.**\n", + "# **To active T4:**\n", + "# **1. click on the \"Runtime\" tab**\n", + "# **2. click on \"Change runtime type\"**\n", + "# **3. select T4 GPU under Hardware Accelerator**\n", + "# **NOTE: there is a weekly usage limit on using T4 for free**\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4vZI1vS6wAO5", + "outputId": "83abeb6a-7415-4a5b-c98f-43123b0740dd" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.3.0-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.124-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (669 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m669.1/669.1 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m52.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.1)\n", + "\u001b[31mERROR: Operation cancelled by user\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 385 + }, + "id": "fNcIsJ7Ewn-S", + "outputId": "37e8883e-8a18-481e-d956-2e342cf4309f" + }, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "errorDetails": { + "actions": [ + { + "action": "open_url", + "actionText": "Open Examples", + "url": "/notebooks/snippets/importing_libraries.ipynb" + } + ] + }, + "evalue": "No module named 'llmware'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mllmware\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetup\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mSetup\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mllmware\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlibrary\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mLibrary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mllmware\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprompts\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mPrompt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mHumanInTheLoop\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mllmware\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretrieval\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mQuery\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'llmware'", + "", + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0;32m\nNOTE: If your import is failing due to a missing package, you can\nmanually install dependencies using either !pip or !apt.\n\nTo view examples of installing some common dependencies, click the\n\"Open Examples\" button below.\n\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n" + ] + } + ], + "source": [ + "import os\n", + "from llmware.setup import Setup\n", + "from llmware.library import Library\n", + "from llmware.prompts import Prompt, HumanInTheLoop\n", + "from llmware.retrieval import Query\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yXk6A_9Zysmv" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "llm = \"llmware/dragon-yi-6b-gguf\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qB6uxOZdywo4" + }, + "outputs": [], + "source": [ + "local_path = Setup().load_sample_files()\n", + "agreements_path = os.path.join(local_path, \"AgreementsLarge\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YCOM8ATYy2l4" + }, + "outputs": [], + "source": [ + "print(f\"\\nStarting: Parsing 'AgreementsLarge' Folder\")\n", + "msa_lib = Library().create_new_library(\"example6_library\")\n", + "msa_lib.add_files(agreements_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VHPA9saYy73o" + }, + "outputs": [], + "source": [ + "print(f\"\\nCompleted Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\")\n", + "q = Query(msa_lib)\n", + "query = '\"master services agreement\"'\n", + "results = q.text_search_by_page(query, page_num=1, results_only=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "nh4JVab_zDbo" + }, + "outputs": [], + "source": [ + "msa_docs = results[\"file_source\"]\n", + "msa_doc_ids = results[\"doc_ID\"]\n", + "prompter = Prompt().load_model(llm)\n", + "print(\"update: identified the following msa doc id: \", msa_doc_ids)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 211 + }, + "id": "iAmOCEC-zdiI", + "outputId": "106d0350-7ab1-41e3-9bf4-30961ae5601c" + }, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'msa_doc_ids' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdoc_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsa_doc_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\n\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mdocs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmsa_docs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msep\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdocs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mNameError\u001b[0m: name 'msa_doc_ids' is not defined" + ] + } + ], + "source": [ + "for i, doc_id in enumerate(msa_doc_ids):\n", + "\n", + " print(\"\\n\")\n", + " docs = msa_docs[i]\n", + " if os.sep in docs:\n", + " docs = docs.split(os.sep)[-1]\n", + "\n", + " print (i+1, \"Reviewing MSA - \", doc_id, docs)\n", + "\n", + " doc_filter = {\"doc_ID\": [doc_id]}\n", + " termination_provisions = q.text_query_with_document_filter(\"termination\", doc_filter)\n", + "\n", + " sources = prompter.add_source_query_results(termination_provisions)\n", + "\n", + " response = prompter.prompt_with_source(\"What is the notice for termination for convenience?\")\n", + "\n", + " stats = prompter.evidence_comparison_stats(response)\n", + " ev_source = prompter.evidence_check_sources(response)\n", + "\n", + " for i, resp in enumerate(response):\n", + " print(\"update: llm response - \", resp)\n", + " print(\"update: compare with evidence- \", stats[i][\"comparison_stats\"])\n", + " print(\"update: sources - \", ev_source[i][\"source_review\"])\n", + "\n", + " prompter.clear_source_materials()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "BSx-KqWizpG4" + }, + "outputs": [], + "source": [ + "print(\"\\nupdate: Prompt state saved at: \", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id))\n", + "prompter.save_state()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mrbGQPCb0Ye3" + }, + "outputs": [], + "source": [ + "csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv()\n", + "print(\"\\nupdate: CSV output for human review - \", csv_output)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_6_rag_multi_step_query_version_2.ipynb b/tutorials/notebooks/fast_start_examples/example_6_rag_multi_step_query_version_2.ipynb new file mode 100644 index 0000000..df30f65 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_6_rag_multi_step_query_version_2.ipynb @@ -0,0 +1,918 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "3PzJTYKBxZHE", + "outputId": "0a79f666-561e-4d26-d25b-ce7de19ba303" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cloning into 'llmware'...\n", + "remote: Enumerating objects: 5664, done.\u001b[K\n", + "remote: Counting objects: 100% (1548/1548), done.\u001b[K\n", + "remote: Compressing objects: 100% (679/679), done.\u001b[K\n", + "remote: Total 5664 (delta 903), reused 1115 (delta 838), pack-reused 4116\u001b[K\n", + "Receiving objects: 100% (5664/5664), 532.23 MiB | 23.42 MiB/s, done.\n", + "Resolving deltas: 100% (3142/3142), done.\n", + "Updating files: 100% (468/468), done.\n", + "/content/llmware\n", + "Processing /content/llmware\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting e\n", + " Downloading e-1.4.5.tar.gz (1.8 kB)\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting boto3==1.24.53 (from llmware==0.2.14)\n", + " Downloading boto3-1.24.53-py3-none-any.whl (132 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.5/132.5 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.20.3)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (1.25.2)\n", + "Collecting openai>=1.0.0 (from llmware==0.2.14)\n", + " Downloading openai-1.30.1-py3-none-any.whl (320 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pymongo>=4.7.0 (from llmware==0.2.14)\n", + " Downloading pymongo-4.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (670 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.0/670.0 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.19.1)\n", + "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (2.2.1+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (4.40.2)\n", + "Collecting Wikipedia-API==0.6.0 (from llmware==0.2.14)\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Collecting psycopg-binary==3.1.17 (from llmware==0.2.14)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware==0.2.14)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware==0.2.14)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware==0.2.14)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Collecting einops==0.7.0 (from llmware==0.2.14)\n", + " Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.10.2.post1)\n", + "Collecting botocore<1.28.0,>=1.27.53 (from boto3==1.24.53->llmware==0.2.14)\n", + " Downloading botocore-1.27.96-py3-none-any.whl (9.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m29.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3==1.24.53->llmware==0.2.14)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.7.0,>=0.6.0 (from boto3==1.24.53->llmware==0.2.14)\n", + " Downloading s3transfer-0.6.2-py3-none-any.whl (79 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware==0.2.14) (4.11.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Wikipedia-API==0.6.0->llmware==0.2.14) (2.31.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (2023.6.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (4.66.4)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (24.0)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->llmware==0.2.14) (1.7.0)\n", + "Collecting httpx<1,>=0.23.0 (from openai>=1.0.0->llmware==0.2.14)\n", + " Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (2.7.1)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (1.3.1)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware==0.2.14)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m34.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (3.1.4)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "Collecting nvidia-nccl-cu12==2.19.3 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", + "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (2.2.0)\n", + "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware==0.2.14) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware==0.2.14) (0.4.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware==0.2.14) (3.7)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware==0.2.14) (1.2.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14) (2.8.2)\n", + "Collecting urllib3<1.27,>=1.25.4 (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14)\n", + " Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14) (2024.2.2)\n", + "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14)\n", + " Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14)\n", + " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware==0.2.14) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware==0.2.14) (4.2.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware==0.2.14) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware==0.2.14) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Wikipedia-API==0.6.0->llmware==0.2.14) (3.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware==0.2.14) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware==0.2.14) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->llmware==0.2.14) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->llmware==0.2.14) (1.3.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware==0.2.14) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14) (1.16.0)\n", + "Building wheels for collected packages: e, llmware\n", + " Building wheel for e (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for e: filename=e-1.4.5-py3-none-any.whl size=2793 sha256=ff3f471fbc464db8eff967277fa643ec0ba3b94c10c1644a6e15f8b01e908626\n", + " Stored in directory: /root/.cache/pip/wheels/3d/dc/3e/208e61209d7750644677d72934d2997177aa34e894bd398e16\n", + " Building wheel for llmware (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for llmware: filename=llmware-0.2.14-py3-none-any.whl size=56031832 sha256=ea11b7471e405abe341532f51f666e25979276045dfeed4123f7667ee2b1c13e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-2tk0k1g5/wheels/ff/a8/4a/2288aab3d94134f03a5bcbe7f6b55020b68ca718e0dc78323b\n", + "Successfully built e llmware\n", + "Installing collected packages: e, urllib3, psycopg-binary, psycopg, pgvector, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, jmespath, h11, einops, dnspython, colorama, pymongo, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpcore, botocore, Wikipedia-API, s3transfer, nvidia-cusolver-cu12, httpx, openai, boto3, llmware\n", + " Attempting uninstall: urllib3\n", + " Found existing installation: urllib3 2.0.7\n", + " Uninstalling urllib3-2.0.7:\n", + " Successfully uninstalled urllib3-2.0.7\n", + "Successfully installed Wikipedia-API-0.6.0 boto3-1.24.53 botocore-1.27.96 colorama-0.4.6 dnspython-2.6.1 e-1.4.5 einops-0.7.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 jmespath-1.0.1 llmware-0.2.14 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 openai-1.30.1 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.2 s3transfer-0.6.2 urllib3-1.26.18\n", + "Collecting faiss-cpu\n", + " Downloading faiss_cpu-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (27.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.0/27.0 MB\u001b[0m \u001b[31m43.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from faiss-cpu) (1.25.2)\n", + "Installing collected packages: faiss-cpu\n", + "Successfully installed faiss-cpu-1.8.0\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install faiss-cpu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mwADpGQ9x1Kd" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.setup import Setup\n", + "from llmware.library import Library\n", + "from llmware.prompts import Prompt, HumanInTheLoop\n", + "from llmware.retrieval import Query\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qisovkCXygD9" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "\n", + "llm = \"llmware/dragon-yi-6b-gguf\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ksZy1eJazIeu" + }, + "outputs": [], + "source": [ + "local_path = Setup().load_sample_files()\n", + "agreements_path = os.path.join(local_path, \"AgreementsLarge\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "20K71ig1zU2T", + "outputId": "23a519f3-7171-4e96-e34f-31ae19857620" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Starting: Parsing 'AgreementsLarge' Folder\n" + ] + }, + { + "data": { + "text/plain": [ + "{'docs_added': 81,\n", + " 'blocks_added': 12552,\n", + " 'images_added': 0,\n", + " 'pages_added': 1345,\n", + " 'tables_added': 10,\n", + " 'rejected_files': []}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(f\"\\nStarting: Parsing 'AgreementsLarge' Folder\")\n", + "msa_lib = Library().create_new_library(\"example6_library\")\n", + "msa_lib.add_files(agreements_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "djgIpiUI0jpF", + "outputId": "6f538534-c4ec-430e-8ad6-bca34e87e3eb" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Completed Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\n" + ] + } + ], + "source": [ + "print(f\"\\nCompleted Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\")\n", + "\n", + "q = Query(msa_lib)\n", + "query = '\"master services agreement\"'\n", + "results = q.text_search_by_page(query, page_num=1, results_only=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cRUgy6YH029j" + }, + "outputs": [], + "source": [ + "msa_docs = results[\"file_source\"]\n", + "msa_doc_ids = results[\"doc_ID\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 176, + "referenced_widgets": [ + "750db138aba04233b7552a809fe028e7", + "73f01dd1f62148ccbc03fd953927076b", + "0c6b1a2f4b6a4602bf5760d9e515f422", + "3bb7c2beaf114d14a0b6e63947c25688", + "58e60eec487640b3a552761db277c105", + "1abee224a04142a798cc9f976fde5f41", + "e96669f232d04c079c91a1121e7dedb5", + "0e66bf7f285343ea98f0456929049e72", + "3a5c51a6dd8e4f8a9fda6f11d370529a", + "93a0447906784d72b7b2b4a5971c66d9", + "a94fb32c43794d6b87f15c22714c68f1" + ] + }, + "id": "doJyVwSQ09iJ", + "outputId": "aa7bf1d0-a0b6-4e74-a000-6a5cb2a3ef79" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "750db138aba04233b7552a809fe028e7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "dragon-yi-6b-q4_k_m.gguf: 0%| | 0.00/3.67G [00:00=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.20.3)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (1.25.2)\n", + "Collecting openai>=1.0.0 (from llmware==0.2.14)\n", + " Downloading openai-1.30.1-py3-none-any.whl (320 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m320.6/320.6 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pymongo>=4.7.0 (from llmware==0.2.14)\n", + " Downloading pymongo-4.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (670 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m670.0/670.0 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.19.1)\n", + "Requirement already satisfied: torch>=1.13.1 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (2.2.1+cu121)\n", + "Requirement already satisfied: transformers>=4.36.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (4.40.2)\n", + "Collecting Wikipedia-API==0.6.0 (from llmware==0.2.14)\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Collecting psycopg-binary==3.1.17 (from llmware==0.2.14)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m17.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware==0.2.14)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware==0.2.14)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware==0.2.14)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Collecting einops==0.7.0 (from llmware==0.2.14)\n", + " Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware==0.2.14) (0.10.2.post1)\n", + "Collecting botocore<1.28.0,>=1.27.53 (from boto3==1.24.53->llmware==0.2.14)\n", + " Downloading botocore-1.27.96-py3-none-any.whl (9.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m29.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3==1.24.53->llmware==0.2.14)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.7.0,>=0.6.0 (from boto3==1.24.53->llmware==0.2.14)\n", + " Downloading s3transfer-0.6.2-py3-none-any.whl (79 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware==0.2.14) (4.11.0)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from Wikipedia-API==0.6.0->llmware==0.2.14) (2.31.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (3.14.0)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (2023.6.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (4.66.4)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (6.0.1)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware==0.2.14) (24.0)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.8.1)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware==0.2.14) (1.0.8)\n", + "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (3.7.1)\n", + "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.0.0->llmware==0.2.14) (1.7.0)\n", + "Collecting httpx<1,>=0.23.0 (from openai>=1.0.0->llmware==0.2.14)\n", + " Downloading httpx-0.27.0-py3-none-any.whl (75 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.6/75.6 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (2.7.1)\n", + "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.0.0->llmware==0.2.14) (1.3.1)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware==0.2.14)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m34.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (3.3)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (3.1.4)\n", + "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n", + "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n", + "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n", + "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n", + "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n", + "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n", + "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n", + "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n", + "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n", + "Collecting nvidia-nccl-cu12==2.19.3 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n", + "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n", + "Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.13.1->llmware==0.2.14) (2.2.0)\n", + "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.13.1->llmware==0.2.14)\n", + " Using cached nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n", + "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware==0.2.14) (2023.12.25)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.36.0->llmware==0.2.14) (0.4.3)\n", + "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware==0.2.14) (3.7)\n", + "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.0.0->llmware==0.2.14) (1.2.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14) (2.8.2)\n", + "Collecting urllib3<1.27,>=1.25.4 (from botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14)\n", + " Downloading urllib3-1.26.18-py2.py3-none-any.whl (143 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14) (2024.2.2)\n", + "Collecting httpcore==1.* (from httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14)\n", + " Downloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.0.0->llmware==0.2.14)\n", + " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware==0.2.14) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware==0.2.14) (4.2.1)\n", + "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware==0.2.14) (0.6.0)\n", + "Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.0.0->llmware==0.2.14) (2.18.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->Wikipedia-API==0.6.0->llmware==0.2.14) (3.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware==0.2.14) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware==0.2.14) (1.16.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.13.1->llmware==0.2.14) (2.1.5)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.13.1->llmware==0.2.14) (1.3.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware==0.2.14) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.28.0,>=1.27.53->boto3==1.24.53->llmware==0.2.14) (1.16.0)\n", + "Building wheels for collected packages: e, llmware\n", + " Building wheel for e (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for e: filename=e-1.4.5-py3-none-any.whl size=2793 sha256=ff3f471fbc464db8eff967277fa643ec0ba3b94c10c1644a6e15f8b01e908626\n", + " Stored in directory: /root/.cache/pip/wheels/3d/dc/3e/208e61209d7750644677d72934d2997177aa34e894bd398e16\n", + " Building wheel for llmware (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for llmware: filename=llmware-0.2.14-py3-none-any.whl size=56031832 sha256=ea11b7471e405abe341532f51f666e25979276045dfeed4123f7667ee2b1c13e\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-2tk0k1g5/wheels/ff/a8/4a/2288aab3d94134f03a5bcbe7f6b55020b68ca718e0dc78323b\n", + "Successfully built e llmware\n", + "Installing collected packages: e, urllib3, psycopg-binary, psycopg, pgvector, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, jmespath, h11, einops, dnspython, colorama, pymongo, nvidia-cusparse-cu12, nvidia-cudnn-cu12, httpcore, botocore, Wikipedia-API, s3transfer, nvidia-cusolver-cu12, httpx, openai, boto3, llmware\n", + " Attempting uninstall: urllib3\n", + " Found existing installation: urllib3 2.0.7\n", + " Uninstalling urllib3-2.0.7:\n", + " Successfully uninstalled urllib3-2.0.7\n", + "Successfully installed Wikipedia-API-0.6.0 boto3-1.24.53 botocore-1.27.96 colorama-0.4.6 dnspython-2.6.1 e-1.4.5 einops-0.7.0 h11-0.14.0 httpcore-1.0.5 httpx-0.27.0 jmespath-1.0.1 llmware-0.2.14 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.127 nvidia-nvtx-cu12-12.1.105 openai-1.30.1 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.7.2 s3transfer-0.6.2 urllib3-1.26.18\n", + "Collecting faiss-cpu\n", + " Downloading faiss_cpu-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (27.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m27.0/27.0 MB\u001b[0m \u001b[31m43.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from faiss-cpu) (1.25.2)\n", + "Installing collected packages: faiss-cpu\n", + "Successfully installed faiss-cpu-1.8.0\n" + ] + } + ], + "source": [ + "!pip install llmware\n", + "!pip install faiss-cpu" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "mwADpGQ9x1Kd" + }, + "outputs": [], + "source": [ + "import os\n", + "from llmware.setup import Setup\n", + "from llmware.library import Library\n", + "from llmware.prompts import Prompt, HumanInTheLoop\n", + "from llmware.retrieval import Query\n", + "from llmware.configs import LLMWareConfig" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qisovkCXygD9" + }, + "outputs": [], + "source": [ + "LLMWareConfig().set_active_db(\"sqlite\")\n", + "\n", + "llm = \"llmware/dragon-yi-6b-gguf\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ksZy1eJazIeu" + }, + "outputs": [], + "source": [ + "local_path = Setup().load_sample_files()\n", + "agreements_path = os.path.join(local_path, \"AgreementsLarge\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "20K71ig1zU2T", + "outputId": "23a519f3-7171-4e96-e34f-31ae19857620" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Starting: Parsing 'AgreementsLarge' Folder\n" + ] + }, + { + "data": { + "text/plain": [ + "{'docs_added': 81,\n", + " 'blocks_added': 12552,\n", + " 'images_added': 0,\n", + " 'pages_added': 1345,\n", + " 'tables_added': 10,\n", + " 'rejected_files': []}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "print(f\"\\nStarting: Parsing 'AgreementsLarge' Folder\")\n", + "msa_lib = Library().create_new_library(\"example6_library\")\n", + "msa_lib.add_files(agreements_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "djgIpiUI0jpF", + "outputId": "6f538534-c4ec-430e-8ad6-bca34e87e3eb" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Completed Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\n" + ] + } + ], + "source": [ + "print(f\"\\nCompleted Parsing - now, let's look for the 'master service agreements', e.g., 'msa'\")\n", + "\n", + "q = Query(msa_lib)\n", + "query = '\"master services agreement\"'\n", + "results = q.text_search_by_page(query, page_num=1, results_only=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cRUgy6YH029j" + }, + "outputs": [], + "source": [ + "msa_docs = results[\"file_source\"]\n", + "msa_doc_ids = results[\"doc_ID\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 176, + "referenced_widgets": [ + "750db138aba04233b7552a809fe028e7", + "73f01dd1f62148ccbc03fd953927076b", + "0c6b1a2f4b6a4602bf5760d9e515f422", + "3bb7c2beaf114d14a0b6e63947c25688", + "58e60eec487640b3a552761db277c105", + "1abee224a04142a798cc9f976fde5f41", + "e96669f232d04c079c91a1121e7dedb5", + "0e66bf7f285343ea98f0456929049e72", + "3a5c51a6dd8e4f8a9fda6f11d370529a", + "93a0447906784d72b7b2b4a5971c66d9", + "a94fb32c43794d6b87f15c22714c68f1" + ] + }, + "id": "doJyVwSQ09iJ", + "outputId": "aa7bf1d0-a0b6-4e74-a000-6a5cb2a3ef79" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:88: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "750db138aba04233b7552a809fe028e7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "dragon-yi-6b-q4_k_m.gguf: 0%| | 0.00/3.67G [00:00=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.144-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.4)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m31.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.2)\n", + "Collecting botocore<1.35.0,>=1.34.144 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.144-py3-none-any.whl (12.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m37.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.2-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.7/82.7 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.15.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m24.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.144->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.144->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.7.4)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.144->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.144 botocore-1.34.144 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.3 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.8.0 s3transfer-0.10.2\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "PqGg_VjFbTX6" + }, + "outputs": [], + "source": [ + "from llmware.models import ModelCatalog" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "3jh9oTQwbVx2" + }, + "outputs": [], + "source": [ + "tools = ModelCatalog().list_llm_tools()\n", + "tool_map = ModelCatalog().get_llm_fx_mapping()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "77b6hZ2ubeza", + "outputId": "e7808eb1-9515-4347-ef86-99e33a5e8eed" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "List of SLIM model tools (GGUF) in the ModelCatalog\n", + "\n", + "0 - tool: ner - model_name: slim-ner-tool - model_family: GGUFGenerativeModel\n", + "1 - tool: sentiment - model_name: slim-sentiment-tool - model_family: GGUFGenerativeModel\n", + "2 - tool: topics - model_name: slim-topics-tool - model_family: GGUFGenerativeModel\n", + "3 - tool: ratings - model_name: slim-ratings-tool - model_family: GGUFGenerativeModel\n", + "4 - tool: emotions - model_name: slim-emotions-tool - model_family: GGUFGenerativeModel\n", + "5 - tool: nli - model_name: slim-nli-tool - model_family: GGUFGenerativeModel\n", + "6 - tool: intent - model_name: slim-intent-tool - model_family: GGUFGenerativeModel\n", + "7 - tool: sql - model_name: slim-sql-tool - model_family: GGUFGenerativeModel\n", + "8 - tool: answer - model_name: bling-answer-tool - model_family: GGUFGenerativeModel\n", + "9 - tool: category - model_name: slim-category-tool - model_family: GGUFGenerativeModel\n", + "10 - tool: tags - model_name: slim-tags-tool - model_family: GGUFGenerativeModel\n", + "11 - tool: summary - model_name: slim-summary-tool - model_family: GGUFGenerativeModel\n", + "12 - tool: xsum - model_name: slim-xsum-tool - model_family: GGUFGenerativeModel\n", + "13 - tool: extract - model_name: slim-extract-tool - model_family: GGUFGenerativeModel\n", + "14 - tool: boolean - model_name: slim-boolean-tool - model_family: GGUFGenerativeModel\n", + "15 - tool: sa-ner - model_name: slim-sa-ner-tool - model_family: GGUFGenerativeModel\n", + "16 - tool: tags-3b - model_name: slim-tags-3b-tool - model_family: GGUFGenerativeModel\n", + "17 - tool: q_gen - model_name: slim-q-gen-tiny-tool - model_family: GGUFGenerativeModel\n", + "18 - tool: qa_gen - model_name: slim-qa-gen-tiny-tool - model_family: GGUFGenerativeModel\n" + ] + } + ], + "source": [ + "print(\"\\nList of SLIM model tools (GGUF) in the ModelCatalog\\n\")\n", + "for i, tool in enumerate(tools):\n", + " model_card = ModelCatalog().lookup_model_card(tool_map[tool])\n", + " print(f\"{i} - tool: {tool} - \"\n", + " f\"model_name: {model_card['model_name']} - \"\n", + " f\"model_family: {model_card['model_family']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rslRQEmvJ-_e" + }, + "source": [ + "# Function calls using different SLIMs" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "1PEKdmTZbkTi", + "outputId": "d2cd3979-fb5c-4d46-b007-68d17a7ff8b1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Executing Function Call Inferences with SLIMs\n", + "\n" + ] + } + ], + "source": [ + "print(\"\\nExecuting Function Call Inferences with SLIMs\\n\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nsbx7C5jRyrF" + }, + "source": [ + "Perform sentiment analysis" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qM3Ub2UBbrWg", + "outputId": "1ec9645e-d3f2-4344-cc1e-aa56e521c46c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "sentiment response: {'sentiment': ['positive']}\n" + ] + } + ], + "source": [ + "passage1 = (\"This is one of the best quarters we can remember for the industrial sector \"\n", + " \"with significant growth across the board in new order volume, as well as price \"\n", + " \"increases in excess of inflation. We continue to see very strong demand, especially \"\n", + " \"in Asia and Europe. Accordingly, we remain bullish on the tier 1 suppliers and would \"\n", + " \"be accumulating more stock on any dips.\")\n", + "\n", + "model = ModelCatalog().load_model(\"slim-sentiment-tool\")\n", + "response = model.function_call(passage1)\n", + "\n", + "print(\"sentiment response: \", response['llm_response'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "eRQ20_Z4R1Z-" + }, + "source": [ + "Named entity recognition" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "is2D0hH3cmDv", + "outputId": "ea4964e2-ec61-4c1c-f964-e749966cdd3c" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ner response: {'people': ['Michael Johnson'], 'location': ['U.S.'], 'organization': [], 'misc': []}\n" + ] + } + ], + "source": [ + "passage2 = \"Michael Johnson was a famous Olympic sprinter from the U.S. in the early 2000s.\"\n", + "\n", + "model = ModelCatalog().load_model(\"slim-ner-tool\")\n", + "response = model.function_call(passage2)\n", + "\n", + "print(\"ner response: \", response['llm_response'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "sgYhLHHSR_IG" + }, + "source": [ + "Extract information based on any parameter" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zL6LTh_lcpU2", + "outputId": "399b83c1-cdd5-40ad-a20e-2a008f8ebbc1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "extract response: {'revenue_growth': ['11% year over year']}\n" + ] + } + ], + "source": [ + "passage3 = (\"Adobe shares tumbled as much as 11% in extended trading Thursday after the design software maker \"\n", + " \"issued strong fiscal first-quarter results but came up slightly short on quarterly revenue guidance. \"\n", + " \"Here’s how the company did, compared with estimates from analysts polled by LSEG, formerly known as Refinitiv: \"\n", + " \"Earnings per share: $4.48 adjusted vs. $4.38 expected Revenue: $5.18 billion vs. $5.14 billion expected \"\n", + " \"Adobe’s revenue grew 11% year over year in the quarter, which ended March 1, according to a statement. \"\n", + " \"Net income decreased to $620 million, or $1.36 per share, from $1.25 billion, or $2.71 per share, \"\n", + " \"in the same quarter a year ago. During the quarter, Adobe abandoned its $20 billion acquisition of \"\n", + " \"design software startup Figma after U.K. regulators found competitive concerns. The company paid \"\n", + " \"Figma a $1 billion termination fee.\")\n", + "\n", + "model = ModelCatalog().load_model(\"slim-extract-tool\")\n", + "response = model.function_call(passage3, function=\"extract\", params=[\"revenue growth\"])\n", + "\n", + "print(\"extract response: \", response['llm_response'])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XLSO67KuSHtk" + }, + "source": [ + "Generate questions, topic, summary, booolean, and tags responses" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 286, + "referenced_widgets": [ + "ce5909ffb35140809cf3d28e697bd820", + "03f517f7c4c041839a73c9a118136a98", + "14248974d27546a9878a58ede2a0824b", + "d891810683914ce1b50bbd984fe72dd2", + "7664f61301a5443eb85683e91ac52066", + "fe5eadc80f7b483f94e712f07fb50d74", + "a8c953b3d5294b8c837d32ef58ab68e4", + "ee20a1af7676472497fa9347a73a3ac2", + "55d9bfaef4fd4f72b81521e9a3229e55", + "b3c59a84b4f9458fb5eefc486e58ab3c", + "f6629315f7ff435baca449d77e9c18c4", + "d0e5607de51d4376af2b8740016fee84", + "5a31f512094f4a4a84df8ed238291360", + "88f930a0843e4a6e9160e1f002685007", + "1c4747678dee4adaad66f9886d58e8f1", + "4e7243be9d99442fac821665f049b9aa", + "c239ddbbcba04b9ba5e27dde88533feb", + "ded92b78613c4a50af7c248c8c0a7efb", + "2503d28eeb354feb865a3494ea7afd5b", + "02e8afc01b91481593cb2ba8989c3488", + "183ee132cf284889ba8dd94a5f490104", + "1167552221e24e4bb3828f1beaddc3af", + "2316a1ade614408ea64f104756a446ef", + "fa87b43386e74fc78ee1005acca09938", + "dcf62b55090141229aed892c9ae9311d", + "e13a1171e54640caa30110d719ea38da", + "4490db7870bf465f831b04d5a04994e7", + "71c5c7f423ed4ed796016b2baf38d51f", + "6326c5c2390741509e08f1656f689eea", + "ab3a661b4f654844b0c73567fd81e7fd", + "fbe235807cca48e2be820e39a7879bb0", + "ac685e9cb17d4b74b77de95fae0a1ffb", + "f39309ea4c71464680b8625ee1179ae1", + "d2ec990c2fea44d191778ce370746335", + "7dc538a4e32c477db5fcb42e9d315cc8", + "17e227a807034b71a379aeab44aca89a", + "19baa88fdb90498da96f152e950d5182", + "5d3f4f83a9cc4fd0a2131e30f37964db", + "fd57dad6def1412e843f14be7fb8e683", + "94815f28c94042a8ba8d9508e9d82edc", + "74c5f136b1fe456fac0bf58fcab97dbd", + "3ca191f2b3144eab885cb129020186e0", + "a1bc48f48bcb40b7a4bc829148b7bb72", + "11935c00579543b4ac86f229ff4856de", + "82518d7d1a3542d2b1baa7834ab73aa8", + "f18214ec5df24a52b7dc203df646d9e7", + "f3a8416bfb0040f4b69d936ef3fe428a", + "3e8437ea4a874538b5225b452c7199d2", + "169ec40c0f4c4abdaebf93193efa64c9", + "faefd9b2820347809d4f667514162bc8", + "a71195cc73f8472dae8b45a35c9dbbd6", + "f4c7d22464e44220ac558568ab01aa0e", + "a1a6126c83e54147b74a2c5a6080b0c4", + "b5d686d855e94a8cbb555391eaebbade", + "2b660070487b48a0abfc605ec81778f2" + ] + }, + "id": "-tPnfKL4v-qD", + "outputId": "3b00f71a-afc2-4c1e-ea46-f3e4f755ba0b" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:llmware.models:ModelCatalog - load_model - fetching model - slim-q-gen-tiny-tool - from remote repository using pull_snapshot_from_hf - this may take a couple of minutes the first time.\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ce5909ffb35140809cf3d28e697bd820", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00 0.8:\n", + " print(\"sentiment is positive with high confidence ... \", sentiment_value, confidence_level)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "00d26355c601472eb0f8a19a32c1bd32": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "02d3bc552bcb4c5b90f0de315be67d93": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9f8ab20a8e294bca8ce34980edc8a552", + "placeholder": "​", + "style": "IPY_MODEL_19fda5135cc0406da73c4971d018d209", + "value": " 4/4 [00:14<00:00,  3.83s/it]" + } + }, + "02e8afc01b91481593cb2ba8989c3488": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "03f517f7c4c041839a73c9a118136a98": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fe5eadc80f7b483f94e712f07fb50d74", + "placeholder": "​", + "style": "IPY_MODEL_a8c953b3d5294b8c837d32ef58ab68e4", + "value": "Fetching 4 files: 100%" + } + }, + "08822554038446c89c6b3c3d7658576b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e7e3af995d594259a5f24406aad341ee", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_98bf9a1696bc4299b828054af22e2d82", + "value": 4 + } + }, + "09fceb5fa71b49378ba23028a22f4d53": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ea2f02f0d53a4a509c87181dd95b153a", + "max": 13849, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_31d1200ef99d486c95bdd5181caf460b", + "value": 13849 + } + }, + "0a91fa8e5d194bc88bdb0a9391b8f4e2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0c4751e0711e4531bcf1712c0c49a810": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0df61654934b4d809513c4f13e29cdd4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0f015639bacf411098f7b4ca37ac2219": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1072a4ce801c4ca4a1af131759132152": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_e9b6730c214a48cd9eeb32cecc555b70", + "placeholder": "​", + "style": "IPY_MODEL_9971cd8a295844a4b72436e4b54dc3c4", + "value": "README.md: 100%" + } + }, + "1167552221e24e4bb3828f1beaddc3af": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "11935c00579543b4ac86f229ff4856de": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "13f1f6d9777841cbaaeeee9f8fda18a7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_82817ccc31e54dc5b25558ae09a179c6", + "IPY_MODEL_ec69580115fd4e949a60d680aad911aa", + "IPY_MODEL_b32f32e2975a41e1a9964c98b832ca7c" + ], + "layout": "IPY_MODEL_ef8bd3dc475b4167b267977d8b6a81e4" + } + }, + "14248974d27546a9878a58ede2a0824b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ee20a1af7676472497fa9347a73a3ac2", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_55d9bfaef4fd4f72b81521e9a3229e55", + "value": 4 + } + }, + "1532a7c6142048b394fc78e979737658": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "169ec40c0f4c4abdaebf93193efa64c9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "17e227a807034b71a379aeab44aca89a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_74c5f136b1fe456fac0bf58fcab97dbd", + "max": 2379, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3ca191f2b3144eab885cb129020186e0", + "value": 2379 + } + }, + "183ee132cf284889ba8dd94a5f490104": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "19baa88fdb90498da96f152e950d5182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a1bc48f48bcb40b7a4bc829148b7bb72", + "placeholder": "​", + "style": "IPY_MODEL_11935c00579543b4ac86f229ff4856de", + "value": " 2.38k/2.38k [00:00<00:00, 29.1kB/s]" + } + }, + "19fda5135cc0406da73c4971d018d209": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "1bcc4a7831844ebc84598fefde3dd73e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_00d26355c601472eb0f8a19a32c1bd32", + "placeholder": "​", + "style": "IPY_MODEL_7d7bc2887eab49ffbf3dd7a1fc0f87b4", + "value": " 1.71G/1.71G [00:20<00:00, 69.3MB/s]" + } + }, + "1c4747678dee4adaad66f9886d58e8f1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_183ee132cf284889ba8dd94a5f490104", + "placeholder": "​", + "style": "IPY_MODEL_1167552221e24e4bb3828f1beaddc3af", + "value": " 1.57k/1.57k [00:00<00:00, 31.5kB/s]" + } + }, + "1db898df5eb045feb90f6c078a7ca903": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_d5a0c23e8713496b97d3e90a1c45c450", + "placeholder": "​", + "style": "IPY_MODEL_d8290cc4e195400cb539eae585c78514", + "value": "config.json: 100%" + } + }, + "1de4a6b923f34c909d52465ca0c2325f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "2039954882e74bf4900a1982362123cf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6c85f00e263248c5a7ce25faa5f2df13", + "placeholder": "​", + "style": "IPY_MODEL_dd72a363ccde43e4b142907fe96a6ef6", + "value": " 1.63k/1.63k [00:00<00:00, 18.1kB/s]" + } + }, + "2125aa9fde814170a2b3cc003f18f783": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2316a1ade614408ea64f104756a446ef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_fa87b43386e74fc78ee1005acca09938", + "IPY_MODEL_dcf62b55090141229aed892c9ae9311d", + "IPY_MODEL_e13a1171e54640caa30110d719ea38da" + ], + "layout": "IPY_MODEL_4490db7870bf465f831b04d5a04994e7" + } + }, + "2503d28eeb354feb865a3494ea7afd5b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "25e21ca487db4c349746414cb8a4aac8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "29cc40c6594641f1bdda693947de55c9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2b660070487b48a0abfc605ec81778f2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2e8a78f3c5b44f71a3e0c0ac727f9f6d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3001c98b5de64d828363b6a5e3f1e293": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_69feb63105d945e5bb43aaced924dc0e", + "placeholder": "​", + "style": "IPY_MODEL_25e21ca487db4c349746414cb8a4aac8", + "value": " 1.86k/1.86k [00:00<00:00, 29.9kB/s]" + } + }, + "30a54d64142949369072a6fed5fe3612": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "31d1200ef99d486c95bdd5181caf460b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "385d024868a847e886df7b0de5b27827": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "38b12e9f780c49458a7bcd85ed083553": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "39b8c54128a743f397a7d43a39961fdd": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_5aa5e5a40932474dbad50781fa6e4711", + "placeholder": "​", + "style": "IPY_MODEL_5031078a749d46038e5e833e46d5d881", + "value": "README.md: 100%" + } + }, + "3ca191f2b3144eab885cb129020186e0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "3cc6885b47c645eb9b259b20794a799e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3cffa10b2502444eb8e57f87f566f3ed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "3e8437ea4a874538b5225b452c7199d2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b5d686d855e94a8cbb555391eaebbade", + "placeholder": "​", + "style": "IPY_MODEL_2b660070487b48a0abfc605ec81778f2", + "value": " 668M/668M [00:09<00:00, 119MB/s]" + } + }, + "3f855dc1493a4e088e6b7c09f0dc1707": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "410e96e19ddb46beb4b2a14a8170ae00": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "41618ecc41984ae391fef3faec13ba36": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_446a3665b47d46319f9d767358edc9f4", + "placeholder": "​", + "style": "IPY_MODEL_30a54d64142949369072a6fed5fe3612", + "value": " 20.9k/20.9k [00:00<00:00, 221kB/s]" + } + }, + "41a9b2d9c2a6467693c6abd39b856cc6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "445f8b445a814d7da8eb57f4bb4078f6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b3c9bfca53f24ff6abd2b2829dffd706", + "IPY_MODEL_8648339315e5488e8572293fa0d0ecab", + "IPY_MODEL_74836482e17442bd996eb63e201b81b0" + ], + "layout": "IPY_MODEL_9f0e9a9f6ebf406fb9a0eae5739ffc9d" + } + }, + "446a3665b47d46319f9d767358edc9f4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4490db7870bf465f831b04d5a04994e7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "45a9e38b7fca469783447235eace2342": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5999ccd426df413b99bc2806f8c50c08", + "IPY_MODEL_79607c545a244aa28e800efb4f2e9939", + "IPY_MODEL_8c1c201328d146289aa0f8b17a6ec166" + ], + "layout": "IPY_MODEL_7b2b5e3c9c4d41e5bec99dcaaf438df2" + } + }, + "4a87d554fe344dc4930ec2f9b4869ae1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4b014bd5b38d4d9380ef8e968d1f2bab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_67da5fe862264f0a9e1cb8cd7a341ed8", + "IPY_MODEL_eb45be721e1749ce98a2bf13747182d3", + "IPY_MODEL_1bcc4a7831844ebc84598fefde3dd73e" + ], + "layout": "IPY_MODEL_fee51587ddcb4568b55477502872a155" + } + }, + "4e7243be9d99442fac821665f049b9aa": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4f6900d3851b415d9422fc72da9b3c74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5031078a749d46038e5e833e46d5d881": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "51d2207e57b64a17a1047687429a0ac1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "55d9bfaef4fd4f72b81521e9a3229e55": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "55e672fc3f9540b392facef6347124fb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "57dd14bf72e641d1a1f2945fdb760581": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5999ccd426df413b99bc2806f8c50c08": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b5a5433d31e745c1ba7c9a565c8a69a9", + "placeholder": "​", + "style": "IPY_MODEL_c8f289c026564c2f9efd42b0ca191d8f", + "value": ".gitattributes: 100%" + } + }, + "5a31f512094f4a4a84df8ed238291360": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c239ddbbcba04b9ba5e27dde88533feb", + "placeholder": "​", + "style": "IPY_MODEL_ded92b78613c4a50af7c248c8c0a7efb", + "value": ".gitattributes: 100%" + } + }, + "5aa5e5a40932474dbad50781fa6e4711": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5c87e66e0d1949dd8dd89dc33b366b7e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5cf26d61321147ffbd1592cea197dbaf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "5d06a9cd80644ddf91e226a6349db0db": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5d3f4f83a9cc4fd0a2131e30f37964db": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "5ed7ec76923d452da6ff321e7b111351": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1072a4ce801c4ca4a1af131759132152", + "IPY_MODEL_64548e4acab94cc4a2fe0c0ff5bb9c0e", + "IPY_MODEL_d91b79e079834ab8914102e7ca0dfaed" + ], + "layout": "IPY_MODEL_df058f04ca974e688de4493eb7d94cbf" + } + }, + "63101a02f53c4b6083be1132fb29dbba": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6326c5c2390741509e08f1656f689eea": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "63dbcabf46524a2f874d764dd4074eb8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "64548e4acab94cc4a2fe0c0ff5bb9c0e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_99e3049714044436bdb06f7c568dd787", + "max": 1724, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_651a627d47e94992842e10dc8574d909", + "value": 1724 + } + }, + "651a627d47e94992842e10dc8574d909": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "663a362c2c364ba9a034486557f9108e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "67a38f85b62347da9075e49f4a75daff": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "67da5fe862264f0a9e1cb8cd7a341ed8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2e8a78f3c5b44f71a3e0c0ac727f9f6d", + "placeholder": "​", + "style": "IPY_MODEL_0a91fa8e5d194bc88bdb0a9391b8f4e2", + "value": "slim-xsum.gguf: 100%" + } + }, + "694f3d7fa83c47e78b55db0c74a02855": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "69feb63105d945e5bb43aaced924dc0e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6c314efdc80d4e118d838c8f1c993c37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "6c85f00e263248c5a7ce25faa5f2df13": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6ca4416686994a559e33b67ab163e9cc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "6f366746731e4f0cb9cd211387120182": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f2114e2a7d874d7fa1333e15cf548fee", + "IPY_MODEL_7f95396470cd46c092484e855d108d7c", + "IPY_MODEL_2039954882e74bf4900a1982362123cf" + ], + "layout": "IPY_MODEL_3cc6885b47c645eb9b259b20794a799e" + } + }, + "71c5c7f423ed4ed796016b2baf38d51f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "734bfcc5284a42bf8f5ec85acc5983ce": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "74836482e17442bd996eb63e201b81b0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8a0a5606e4924a329636dcb693e40d34", + "placeholder": "​", + "style": "IPY_MODEL_7721c1c413bc4a888704a424a6ea9f56", + "value": " 4/4 [00:20<00:00,  5.53s/it]" + } + }, + "74c5f136b1fe456fac0bf58fcab97dbd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "74d1a425aee34a738a67b82f5238aeb7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_c91ad74380e7462fb6b08e33be7abeb2", + "max": 1864, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_0c4751e0711e4531bcf1712c0c49a810", + "value": 1864 + } + }, + "7664f61301a5443eb85683e91ac52066": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7721c1c413bc4a888704a424a6ea9f56": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "792f998dec2c4a22b8be7db83b4edba7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "79607c545a244aa28e800efb4f2e9939": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a28a3d68e82e4ea0b844422939b79a3c", + "max": 1632, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_3cffa10b2502444eb8e57f87f566f3ed", + "value": 1632 + } + }, + "7a4eb5bda2d34d1fa8fc20d32c412dfc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7af7ee46b91c4c049d08eff417b8562c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7b2b5e3c9c4d41e5bec99dcaaf438df2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7b568521b639484a87ec7a3d4eeddef0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7d7bc2887eab49ffbf3dd7a1fc0f87b4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "7dc538a4e32c477db5fcb42e9d315cc8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fd57dad6def1412e843f14be7fb8e683", + "placeholder": "​", + "style": "IPY_MODEL_94815f28c94042a8ba8d9508e9d82edc", + "value": "README.md: 100%" + } + }, + "7f95396470cd46c092484e855d108d7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_63101a02f53c4b6083be1132fb29dbba", + "max": 1629, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_6c314efdc80d4e118d838c8f1c993c37", + "value": 1629 + } + }, + "82518d7d1a3542d2b1baa7834ab73aa8": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f18214ec5df24a52b7dc203df646d9e7", + "IPY_MODEL_f3a8416bfb0040f4b69d936ef3fe428a", + "IPY_MODEL_3e8437ea4a874538b5225b452c7199d2" + ], + "layout": "IPY_MODEL_169ec40c0f4c4abdaebf93193efa64c9" + } + }, + "82817ccc31e54dc5b25558ae09a179c6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_663a362c2c364ba9a034486557f9108e", + "placeholder": "​", + "style": "IPY_MODEL_5c87e66e0d1949dd8dd89dc33b366b7e", + "value": "slim-topics.gguf: 100%" + } + }, + "83610e7e82e740aa90ae1653ac2b5b3b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_39b8c54128a743f397a7d43a39961fdd", + "IPY_MODEL_74d1a425aee34a738a67b82f5238aeb7", + "IPY_MODEL_3001c98b5de64d828363b6a5e3f1e293" + ], + "layout": "IPY_MODEL_e31afc8380bb4901829c175fc7a12636" + } + }, + "8648339315e5488e8572293fa0d0ecab": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_bd8cc1f1b04a48d99aad217b8e713d7f", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_57dd14bf72e641d1a1f2945fdb760581", + "value": 4 + } + }, + "88f930a0843e4a6e9160e1f002685007": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2503d28eeb354feb865a3494ea7afd5b", + "max": 1566, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_02e8afc01b91481593cb2ba8989c3488", + "value": 1566 + } + }, + "8a0a5606e4924a329636dcb693e40d34": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8c1c201328d146289aa0f8b17a6ec166": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_aba43a36ac224495a32a6aef06f7e24f", + "placeholder": "​", + "style": "IPY_MODEL_aa66e7873ed74f9ea9806e08aca85309", + "value": " 1.63k/1.63k [00:00<00:00, 32.7kB/s]" + } + }, + "94815f28c94042a8ba8d9508e9d82edc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "96b7cac4b68e4ea788f50ad631ba1ac1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "977629094f9a4d2ab428ad59f6f3b266": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ca696d2a1d314be9b6f417c700218335", + "placeholder": "​", + "style": "IPY_MODEL_385d024868a847e886df7b0de5b27827", + "value": "config.json: 100%" + } + }, + "98bf9a1696bc4299b828054af22e2d82": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "9971cd8a295844a4b72436e4b54dc3c4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "99e3049714044436bdb06f7c568dd787": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9f0e9a9f6ebf406fb9a0eae5739ffc9d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9f8ab20a8e294bca8ce34980edc8a552": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a1a6126c83e54147b74a2c5a6080b0c4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a1bc48f48bcb40b7a4bc829148b7bb72": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a28a3d68e82e4ea0b844422939b79a3c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a47311ab6e7c4502a9cd2ecd70542b83": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7a4eb5bda2d34d1fa8fc20d32c412dfc", + "placeholder": "​", + "style": "IPY_MODEL_3f855dc1493a4e088e6b7c09f0dc1707", + "value": "tokenizer_tl.json: 100%" + } + }, + "a71195cc73f8472dae8b45a35c9dbbd6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "a8c953b3d5294b8c837d32ef58ab68e4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "aa66e7873ed74f9ea9806e08aca85309": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ab3a661b4f654844b0c73567fd81e7fd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "aba43a36ac224495a32a6aef06f7e24f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ac685e9cb17d4b74b77de95fae0a1ffb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "acf7cdbca28b469fabf0968d777cdbfa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7af7ee46b91c4c049d08eff417b8562c", + "max": 20900, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_fd8c00bc3fb14c0fb265e9259cac5bac", + "value": 20900 + } + }, + "adf4aa108291431ab5efd995ac55974f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b1d12219e2024648892def9a1e3d76d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_63dbcabf46524a2f874d764dd4074eb8", + "placeholder": "​", + "style": "IPY_MODEL_734bfcc5284a42bf8f5ec85acc5983ce", + "value": "Fetching 4 files: 100%" + } + }, + "b32f32e2975a41e1a9964c98b832ca7c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_0f015639bacf411098f7b4ca37ac2219", + "placeholder": "​", + "style": "IPY_MODEL_7b568521b639484a87ec7a3d4eeddef0", + "value": " 669M/669M [00:13<00:00, 48.5MB/s]" + } + }, + "b3c59a84b4f9458fb5eefc486e58ab3c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b3c9bfca53f24ff6abd2b2829dffd706": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4f6900d3851b415d9422fc72da9b3c74", + "placeholder": "​", + "style": "IPY_MODEL_6ca4416686994a559e33b67ab163e9cc", + "value": "Fetching 4 files: 100%" + } + }, + "b5a5433d31e745c1ba7c9a565c8a69a9": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b5d686d855e94a8cbb555391eaebbade": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "bc4a8e6f47c74225bbe60a973931a4f7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_a47311ab6e7c4502a9cd2ecd70542b83", + "IPY_MODEL_fdc0ade930e34c1fb16b169cdfb4fafa", + "IPY_MODEL_f898f81465bf4c39bce1a57a032cb5a6" + ], + "layout": "IPY_MODEL_96b7cac4b68e4ea788f50ad631ba1ac1" + } + }, + "bd8cc1f1b04a48d99aad217b8e713d7f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c239ddbbcba04b9ba5e27dde88533feb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c8f289c026564c2f9efd42b0ca191d8f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c91ad74380e7462fb6b08e33be7abeb2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ca696d2a1d314be9b6f417c700218335": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ce5909ffb35140809cf3d28e697bd820": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_03f517f7c4c041839a73c9a118136a98", + "IPY_MODEL_14248974d27546a9878a58ede2a0824b", + "IPY_MODEL_d891810683914ce1b50bbd984fe72dd2" + ], + "layout": "IPY_MODEL_7664f61301a5443eb85683e91ac52066" + } + }, + "d0e5607de51d4376af2b8740016fee84": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_5a31f512094f4a4a84df8ed238291360", + "IPY_MODEL_88f930a0843e4a6e9160e1f002685007", + "IPY_MODEL_1c4747678dee4adaad66f9886d58e8f1" + ], + "layout": "IPY_MODEL_4e7243be9d99442fac821665f049b9aa" + } + }, + "d2ec990c2fea44d191778ce370746335": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_7dc538a4e32c477db5fcb42e9d315cc8", + "IPY_MODEL_17e227a807034b71a379aeab44aca89a", + "IPY_MODEL_19baa88fdb90498da96f152e950d5182" + ], + "layout": "IPY_MODEL_5d3f4f83a9cc4fd0a2131e30f37964db" + } + }, + "d576575f0add44689294a784cb61884d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_977629094f9a4d2ab428ad59f6f3b266", + "IPY_MODEL_09fceb5fa71b49378ba23028a22f4d53", + "IPY_MODEL_ef39b1f4495d4e0593a899b32453735e" + ], + "layout": "IPY_MODEL_51d2207e57b64a17a1047687429a0ac1" + } + }, + "d5a0c23e8713496b97d3e90a1c45c450": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "d8290cc4e195400cb539eae585c78514": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "d891810683914ce1b50bbd984fe72dd2": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b3c59a84b4f9458fb5eefc486e58ab3c", + "placeholder": "​", + "style": "IPY_MODEL_f6629315f7ff435baca449d77e9c18c4", + "value": " 4/4 [00:10<00:00,  2.73s/it]" + } + }, + "d91b79e079834ab8914102e7ca0dfaed": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_792f998dec2c4a22b8be7db83b4edba7", + "placeholder": "​", + "style": "IPY_MODEL_5d06a9cd80644ddf91e226a6349db0db", + "value": " 1.72k/1.72k [00:00<00:00, 13.2kB/s]" + } + }, + "dcf62b55090141229aed892c9ae9311d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ab3a661b4f654844b0c73567fd81e7fd", + "max": 38761, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_fbe235807cca48e2be820e39a7879bb0", + "value": 38761 + } + }, + "dd72a363ccde43e4b142907fe96a6ef6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ded92b78613c4a50af7c248c8c0a7efb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "df058f04ca974e688de4493eb7d94cbf": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e13a1171e54640caa30110d719ea38da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ac685e9cb17d4b74b77de95fae0a1ffb", + "placeholder": "​", + "style": "IPY_MODEL_f39309ea4c71464680b8625ee1179ae1", + "value": " 38.8k/38.8k [00:00<00:00, 989kB/s]" + } + }, + "e1a1bd6edc0a4a76808cf2212b1f1f93": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1db898df5eb045feb90f6c078a7ca903", + "IPY_MODEL_acf7cdbca28b469fabf0968d777cdbfa", + "IPY_MODEL_41618ecc41984ae391fef3faec13ba36" + ], + "layout": "IPY_MODEL_1532a7c6142048b394fc78e979737658" + } + }, + "e31afc8380bb4901829c175fc7a12636": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e7e3af995d594259a5f24406aad341ee": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e9b6730c214a48cd9eeb32cecc555b70": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ea2f02f0d53a4a509c87181dd95b153a": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eb45be721e1749ce98a2bf13747182d3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2125aa9fde814170a2b3cc003f18f783", + "max": 1708595136, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_5cf26d61321147ffbd1592cea197dbaf", + "value": 1708595136 + } + }, + "ec69580115fd4e949a60d680aad911aa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4a87d554fe344dc4930ec2f9b4869ae1", + "max": 668787680, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_1de4a6b923f34c909d52465ca0c2325f", + "value": 668787680 + } + }, + "ee20a1af7676472497fa9347a73a3ac2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ef39b1f4495d4e0593a899b32453735e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_29cc40c6594641f1bdda693947de55c9", + "placeholder": "​", + "style": "IPY_MODEL_694f3d7fa83c47e78b55db0c74a02855", + "value": " 13.8k/13.8k [00:00<00:00, 284kB/s]" + } + }, + "ef8bd3dc475b4167b267977d8b6a81e4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f18214ec5df24a52b7dc203df646d9e7": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_faefd9b2820347809d4f667514162bc8", + "placeholder": "​", + "style": "IPY_MODEL_a71195cc73f8472dae8b45a35c9dbbd6", + "value": "q_gen.gguf: 100%" + } + }, + "f2114e2a7d874d7fa1333e15cf548fee": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_55e672fc3f9540b392facef6347124fb", + "placeholder": "​", + "style": "IPY_MODEL_41a9b2d9c2a6467693c6abd39b856cc6", + "value": ".gitattributes: 100%" + } + }, + "f39309ea4c71464680b8625ee1179ae1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f3a8416bfb0040f4b69d936ef3fe428a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f4c7d22464e44220ac558568ab01aa0e", + "max": 667814496, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a1a6126c83e54147b74a2c5a6080b0c4", + "value": 667814496 + } + }, + "f4c7d22464e44220ac558568ab01aa0e": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f6629315f7ff435baca449d77e9c18c4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "f898f81465bf4c39bce1a57a032cb5a6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_410e96e19ddb46beb4b2a14a8170ae00", + "placeholder": "​", + "style": "IPY_MODEL_38b12e9f780c49458a7bcd85ed083553", + "value": " 1.84M/1.84M [00:00<00:00, 5.54MB/s]" + } + }, + "fa87b43386e74fc78ee1005acca09938": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_71c5c7f423ed4ed796016b2baf38d51f", + "placeholder": "​", + "style": "IPY_MODEL_6326c5c2390741509e08f1656f689eea", + "value": "config.json: 100%" + } + }, + "faefd9b2820347809d4f667514162bc8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fbe235807cca48e2be820e39a7879bb0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "fd57dad6def1412e843f14be7fb8e683": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fd8c00bc3fb14c0fb265e9259cac5bac": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "fdc0ade930e34c1fb16b169cdfb4fafa": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_67a38f85b62347da9075e49f4a75daff", + "max": 1842767, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_adf4aa108291431ab5efd995ac55974f", + "value": 1842767 + } + }, + "fe5eadc80f7b483f94e712f07fb50d74": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fee51587ddcb4568b55477502872a155": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ff87a8e4b5904e2ca6d138500603e4cb": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_b1d12219e2024648892def9a1e3d76d4", + "IPY_MODEL_08822554038446c89c6b3c3d7658576b", + "IPY_MODEL_02d3bc552bcb4c5b90f0de315be67d93" + ], + "layout": "IPY_MODEL_0df61654934b4d809513c4f13e29cdd4" + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_examples/example_8_agents.ipynb b/tutorials/notebooks/fast_start_examples/example_8_agents.ipynb new file mode 100644 index 0000000..f894d39 --- /dev/null +++ b/tutorials/notebooks/fast_start_examples/example_8_agents.ipynb @@ -0,0 +1,15872 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "yuurmT9VeEbZ" + }, + "source": [ + "### **If you are using Colab for free, we highly recommend you active the T4 GPU hardware accelerator. Our models are designed to run with at least 16GB of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed to the 13GB Colab gives automatically.**\n", + "### **To active T4:**\n", + "### **1. click on the \"Runtime\" tab**\n", + "### **2. click on \"Change runtime type\"**\n", + "### **3. select T4 GPU under Hardware Accelerator**\n", + "### **NOTE: there is a weekly usage limit on using T4 for free**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "p-3xdQQReK1R" + }, + "source": [ + "# **Fast Start Example 8 - Agents** #\n", + "This example shows how to build locally-running Agents deploying multiple small specialized function-calling models as tools with an integrated work management, process and journaling capability using the LLMfx class.\n", + "\n", + "This example shows a workflow receiving a customer transcript, and having an agent run through a series of analytical and classification steps.\n", + "\n", + " 1. Create an agent using the LLMfx class.\n", + " 2. Load multiple specialized tools for the agent.\n", + " 3. Execute a series of function-calls.\n", + " 4. Generate a consolidated automatic dictionary report.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZZeZz22aaxI0" + }, + "source": [ + "# Create agent and load tools" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "a4nBJr5SdVi6", + "outputId": "2612cc14-1e33-4964-ac7f-89bce5fdfc83" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting llmware\n", + " Downloading llmware-0.3.3-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m12.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.146-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.5)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m48.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m23.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.2)\n", + "Collecting botocore<1.35.0,>=1.34.146 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.146-py3-none-any.whl (12.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m39.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.2-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.7/82.7 kB\u001b[0m \u001b[31m9.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.15.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m29.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.146->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.146->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.7.4)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.146->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.146 botocore-1.34.146 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.3 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.8.0 s3transfer-0.10.2\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "mYZw-HAr3HjW" + }, + "outputs": [], + "source": [ + "from llmware.agents import LLMfx" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "4CqHWBz83I_x", + "outputId": "d71f2b76-3421-4227-ef45-59560a9d894c" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:llmware.agents:Agent - Setting up LLMWare Workspace.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "update: Launching LLMfx process\n", + "step - \t1 - \tcreating object - ready to start processing.\n", + "step - \t2 - \tloading new processing text - 1 new entries\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'text': 'My name is Michael Jones, and I am a long-time customer. The Mixco product is not working currently, and it is having a negative impact on my business, as we can not deliver our products while it is down. This is the fourth time that I have called. My account number is 93203, and my user name is mjones. Our company is based in Tampa, Florida.',\n", + " 'file_source': 'NA',\n", + " 'page_num': 'NA'}]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agent = LLMfx()\n", + "\n", + "customer_transcript = \"My name is Michael Jones, and I am a long-time customer. \" \\\n", + " \"The Mixco product is not working currently, and it is having a negative impact \" \\\n", + " \"on my business, as we can not deliver our products while it is down. \" \\\n", + " \"This is the fourth time that I have called. My account number is 93203, and \" \\\n", + " \"my user name is mjones. Our company is based in Tampa, Florida.\"\n", + "\n", + "agent.load_work(customer_transcript)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QuPgiTmKD8Hm" + }, + "source": [ + "Load individual SLIM tool or multiple tools list" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "c50e606ff8ea41bfb369f6ccf368d238", + "dc19d0fc37b44551b09347ad98bcd854", + "ce2dd158480e4e3e9228298e1e344183", + "a35431bd675a45e8ab6600f8b30241eb", + "30b37ee7444147c296aab9f6bebd38c2", + "50a28bb5c8bb4b4bb23648d4ce2efd1e", + "599f350babe24f0e855b6641b02ce382", + "01383b4d8b0e497889209bca3cdf4003", + "c02ef09824b34a40a83a02964437c4e1", + "083747ed0b384578907784bdc484ebec", + "bed52dd2e4b44da3a46ae1fa0aef8b4a", + "649f1a0e43e040249cfaaece29a921eb", + "43788bb08f5142539ab8f41027eaf542", + "9eb0944256a84d988d5165aa36766492", + "c608cc815d46489b9b76b99241a27187", + "77cc7502d21740b4aa92102284b109e5", + "2f6e2ee4a0e7462ea511a684d456b715", + "20f4709ff0d540abb087cb014d7b39e6", + "af7cc17cdce0495c86c4ef469dce280d", + "1c507f2394d8401f9ff96a7e0f4b04bf", + "ce87e78b4bad496998f17d3810ab4803", + "80abe9b8eb6b40e4ab51ddb158666e38", + "8903cd5eab8848ccacb4f2483af6fe09", + "109f72250d48480d9716035056c3c5b6", + "a3f49a5b7077483d8ab6b3df787a8f69", + "efaf53eee00e464d8b7516347c487b6c", + "6bf4800a104f496580551d7780663d91", + "0ebc0da55c864ddf95063d6a7d7b6c86", + "28ff17a9582e4c7ea3041d4541d72722", + "09c1201403e4414290828f192b11da1b", + "367057b5187e45f48c6f00d04bb43924", + "e8b65730fd5e40a89bacefd21dae5c59", + "76198af1695046078f0f9d5d6499a521", + "db49ca4e13f146c9a4e13e3fb7a12de0", + "5975818abffd49c1bfbb6b6d5045f8a1", + "0fd5e50810a94130a6c73f8d2eb04da4", + "6e292e9a36ac4d2f8ee1a4e8c8aa6a55", + "f133541bdafb409b8640173e35e92b9c", + "6e69b8cebff345538a2c962c1b26b544", + "d6e3a7c11e22444197dddf924f8755c0", + "a1504d98f5d7417593213f4ae38bbf82", + "e30c239447a045e5b465c74b84e36c81", + "359ef70913224e53897ade5c8190fbc3", + "1cf37abf4371432799adbca77e14113f", + "2c2dc9b09f204508a6f0f6f2e2b74997", + "b47da6f3bd63408f866fc31b136fa108", + "655e60d0a1aa4c0991e2ad32429ed723", + "ee725a2e933244c29e4482214a6d878c", + "86d4560216d2498691872d4cc31b0a54", + "1633223262a543099a7c123486c10cae", + "c597d598ae1b4dc682ca706bff7b003f", + "9f02204392e445a9996415243a5df73e", + "1a7fe6874ca14a3eb8c1edde86e9ab2a", + "e8e094d25f14436a8bb34094c98be4c9", + "864b12898ab843318fe4fc17d135a57c", + "e36ee9b5bd324c01a69aec529d8d93a7", + "b5c29069f20349c49f24c15b904112a8", + "c108d6a2f8a9432181f9e0fb97101d79", + "337d812587574da6bc27cb2b93f8d77c", + "5d7842d6a44547198cd517541ac70081", + "d78a01c19bdd452ea4f7776cefaa528d", + "03af8419720d47f0b2f0bf8dfe210694", + "5403d9016ad24b4f984950b6dab7293c", + "88c0e26f6782457596acf7dad841da82", + "89bce38dec4b46e3ba0fc5b83b4dd2c8", + "8606065d14a641e6a4436d6b47e37e1d", + "9038edccd14948b4b38c07d92c963053", + "4597036facca4fdb92c749c5e8f4c388", + "0941e5f976aa4e2b8a1c61982e899f1e", + "73e4aaf3f5404d1e8e55d3efcd91f546", + "136f274c30844df9a35dc0f0949ce5d8", + "63768f1fd5344d6e8807db52f321e9e1", + "2f8aab95ad134805bbaaa89d008f8a8f", + "f3fcf2e21c0c4343a549f9b8830d75b9", + "85e32aa9ee7e448b9530194dbf8e2a9a", + "88c98e0ca9654e85ac5d9ee7a14e0616", + "bf6f6246d6ea4612b8a4d3999c5e7e27", + "cb3b917992324c0dadbeb84f5355327a", + "b18959ab077748c3a31f5bf891855861", + "6ed368d8f4504f0f8eab2c03f5d39bf6", + "1ae20d77c85b443fb38ac4c65052e789", + "c8062d1b311d49b6b4c1356b7947f666", + "88d4399c661d4e959df822cad1c40c0d", + "da7437930c694188a39e87ed9485f5f2", + "c6b1aded975149df8b2ecb9179d40e01", + "84c9cfc683b448dba8f2938dbc26cc92", + "2c307284c72c4ed1b80b35f868de531e", + "8126cc9154df405a8ffb0ccc39c55a6e", + "289163c444ca410591a39e4d58ce66c3", + "cd378d4353f44183a5a973498d57a3a3", + "f51ca89d7d4e48edb248e105855c00f4", + "63a55875b5d1414dac0b23aa0d6aa41e", + "cfaa01fedbb24ddc8ed37d8e6ea2d7b5", + "2fdbd88668014b19afbf709eaac9d475", + "45c15728aa5c4c218c141dd04f6982c1", + "787c0c854443491e96a6022b674cd036", + "d06bbebd55de4b3e897cff863df52f0c", + "9dc5cfc801e64609b5df0a5c786cc4f6", + "29fb62eadfc348149d66fc1878e959b6", + "7b147f5fe70642f3aab7440ca3cb6f81", + "df33745d88f44a0fb2e1c9c4d50d78ab", + "1e79c8ab538c464e955fd96e04828679", + "88647704de064f30a6b6231e7847e9d2", + "eaaebde00dde4d6faba200a2d6e1e108", + "c70a4b8757144004b03168988e616740", + "ca7491e180de4ebb945298d738e9e859", + "fcc1cb133f814268aea4e5d5819a2be5", + "5a1de5963b864ca588cdd1b117ab78c6", + "63d5e43596f940318eeb5af70cd23021", + "0c24cd383c7c4b40a516b06b1f318d8e", + "9d4bee213c2d49b9bb7b368fbb0de8fc", + "89400541c03e4d02b2a58e07cb638e3b", + "92282a9f42e54123b1c1846ad3bbf397", + "17dcff78eb7d43a8ad7eefbc857efb3a", + "f12674ee3b52473baa7ac580ab1bf065", + "0559ecc4bf03490796ff449a7b09a1b0", + "496122e633af408095ad54c5cbbe1719", + "f86c6cb2baa543aea923984f1a602f3f", + "133ea734ecd849d4bb0d1ec493536ef4", + "91014b8fa7db440f934b804625c1da50", + "5a8c55971a1d4cf5b4781a3b1ba13387", + "557f25962c71458fac4ed58c5f7b47aa", + "c030db45bee4459a8959e4be3cd2bf06", + "48f5e8b463bc404fbe06f6402a14c32e", + "6df51addbd994dbcbce1db5e441ab818", + "96bd9b0b906646bf8724ab71222911dd", + "41c9990095c74b9bb953db6f3156d8f3", + "87de6ed49f3349268d5cc126a17da00f", + "7ec64f24665741199ea6babb98e6bd20", + "2741d44c8e0f49898b91ade95a02c406", + "f5ea37958ddc439d9961851c2135b155", + "fda529c056fc48b5bd9aa9d63ffaffbf", + "c32c4eeb2dac41a7a3af225004e26ee9", + "6e65dd613d824cddad36a5886d2cdfc2", + "1cf420542fd24204bbf045614375a658", + "f44edc0e3b34470681d0cc5d5f0638ff", + "d1f83250aeb5489abf43329d00aae2a0", + "64a5ff269d68481a82459eae447a51b1", + "fc44fb1b74644e42b53125ec1b5ab88a", + "68c26624971a43f4951d291d187744fe", + "5cf4e2a351094a8897f747a6c5be35a4", + "9d270fc14a7543d2b1e11c0029d1d4b7", + "8a44d2a1039e44bc95cc57e22092b764", + "20049e23b6fd428684bb285e7ace9cdb", + "c8aca1b8769640158f1434a8757a4f87", + "848b7ca874c04cb18304a7910883d2c7", + "af0e6056aa6a4959ad0f16f96556e0a1", + "a94e1a3df0fc444b8768662d5972c413", + "8707c21348f14e7da8107b946eb34522", + "dfb65319911c45f987718305f5694e31", + "19a4792b33af4a9a9c80de1703e02867", + "8657cc269cca4345a58ea35e57639d79", + "811d696c5d7b4967bf48cbe697826009", + "c189361845264711b0d0c79fad45af0d", + "b65402522dcd4ec7972c1592674450f4", + "b5d7c4471a874e10a0f1b693c93bcc82", + "bf942a11949b42f8a07ae6809c899e14", + "547175aa6ba74dc49e4863fa899b4ac1", + "5e70cc17655346f4a7ed3743a3866699", + "4ce0a312b50f4fb691ede750a43a5fd1", + "a16906eb30c348de908be9d89395e9f7", + "7ac47395c1c74f0095ead32802394a00", + "59d3f0608e5740b68c2d3935ab8511a3", + "b9866abf84cf4176976f79ee98bf3f98", + "3c692b6b6ffa45f7b404e436f701a672", + "fb2ea20cf5ed4f6f9e1d57defa8b9119", + "b31881c5eae84e1a818839e002ecfc56", + "60ac52ed40ca4472852a87837a7d5d5e", + "c86a78582dc046da816a0a28d1ee4933", + "0e001faa2c9140e0a14d6d6d525063c7", + "07ace54eb4324caaa433bc5543eda0b8", + "ab0076031aa3469ba3f54b3134f37ed7", + "219b600056884dcaa45253ef77ac7153", + "0c425fe90f6a4086b917183867a68516", + "ea2f95784b2a4f6ea7de27e890d76666", + "fc4ff33590b84dceb28a2324da5a6500", + "0cf2636cff904136a5b34d32c880b665", + "c0d4c095ce8c49a58aff2cbb5c03ae61", + "e1fe568017974b31a324ae9ea0dc529a", + "9b502e1f6656406696d6b6076b052c22", + "77e957d0ae8f409a878747140352d2bc", + "46951a4442c64006ab20647685e99789", + "a6dad63db3ae4aeb84505091d6050fd4", + "388a2f1a60a9452392a0839edc1adbdb", + "30732727e3ed4312b39fc027878016fe", + "f0727ffb1a6243ea96f3d4a6773ee4a8", + "3f1952a542c34a0d912c92df0d55c2f8", + "f923e8ca320547d5b0d060e48d5ecdd6", + "8a52fa45be4441cc8c13647dcd6fde08", + "8ccc83dbf7764f7db5372d939b3747f4", + "979731627bba4d7d8435fc4427ae1251", + "14800a935ff9444f9162a4c44e944599", + "55b68f9645b34fe08528cfb3eded7b2b", + "cbcbe9299f2b4cdf8b9e51f84e28a658", + "86cdf293240d4d24beab7b6712abfa84", + "0bf7089ae8234aa4af896a61ed6cf92d", + "bb1d8f781b6c4d1da5ede823771e349c", + "56f53277381d4ebeb566bee317ba25b0", + "d3a97e8eb12046bb80331910dcca903d", + "e2f5e9c6b1e5423f966dd13314089bc0", + "2c89ed461d4943c0af077dd4b7139926", + "3a7c34a67fdd4cecae4441809498e236", + "965fbaeb5c9f4906af6ab4b1a56ad3ec", + "9fc0563a97c048d4b54ea9d3ef82bfdf", + "1cbe32437d924ff9a68d04563e325ca6", + "3d1d2e6c5c9f4d25b21e51d3baaf9cdc", + "2911204997a24959b2ed735476a9db72", + "e60a003012a74122bee07c2fde184550", + "1c269e0188ee4d77826f88943dcb0494", + "7e454cf9ec174950a1f03462d61c6be6", + "52589be29d8842f6a4d6c4f122262be4", + "066c745e209a4fae913ac5f0ce2467cb", + "396684b6151945f2a342cf359faff93b", + "62be8d9d7a974d9e9a55e9cd482614d1", + "179d25136b38425fb7e91ae9a0ee1edf", + "582f76ea328b4961ab82185bdcb3b8c3", + "9595616827e546288de274c4195feabf", + "21cf764713c843ca96da432a7525a020", + "9c618665ceb34b83858531b1193ce473", + "ccf39efe6a2c43449e76eedc81b16681", + "db1f23e7ce1c447e9cf0a6ab7e2e114f", + "6830256eb96e42888fe327c12414631b", + "b57c56b169f14695aa6df072aea3ff55", + "5acbebadb7ae41b9bf03b9f4822ae67d", + "db85257e697243dba93a3c3b8f150802", + "780b6721eb984f88aca4fd2388e0cdfe", + "41c44704b7664d05805fea083e546745", + "a1fce0b5541048e0940c376c340f3c80", + "278249a83e13459284de6883a5b7e512", + "1a8083a67b564bb4b34fe997d8b15207", + "49dcd00a0892437fad81cc402240eba1", + "206dd102490d416cb9b16c72ed0f12ff", + "63398c2bd80f41fd9c3dd2c2386983e1", + "f89c46290e2d40b1a27633871fea1034", + "f5b538f368d544d98edf75669d2d6ee8", + "3363da2edd94464490db9c91b1408b89", + "22421464b4974f48b32bb7c84c767070", + "5abfdca4ec7a471e9d1a794eb3ad7260", + "75242ff85af0400499a5e7619c0fa7ce", + "9ba31d8f0b104927bbd7c2f276c8dcbd", + "3d23fafc164e4f6d89d3fc9ed5ff3e41", + "a3098eb16db3407392b619e4148c655a", + "8c4b5d738e3f42dcb6c007891441593e", + "143b90073e1e4e568889c8d06e835a4d", + "bd25df9538214c03b61c044c9ac052ba", + "580d312a73154d3f93b5d140d8763ae0", + "f66e68b5ff504e17b6d6a27b6a6ec108", + "6c6e6ab12e23422f889185a11b05f4b4", + "923e5a8d5bf14f1598b4821119ee4f9b", + "169cbc5706cf4070b2b34da61a8d8550", + "dfb58c209a234fc29291c05151586812", + "8bb72ecb3b264d448f9a1ed58e63173d", + "4f41490b36324765ba601b9eea1be8a8", + "b4b7776484b4488993bc980caa955065", + "89d9ea79d89e49c0aabceb0ace57e5db", + "4653dac509a942ee8c2dfedefc38b839", + "d7c51e1cd394459ba1d3d4f939b1c50b", + "b10b36366ab243299b12bcb78979069f", + "da8e160081584fef98bed42763663476", + "16e628ac719642eda967ce0fb79f6870", + "c898c6941f8b45cfa5016b0358b2ff19", + "15450fe2ce004e9aa6b9006205a99575", + "cf30d63d859c45c5b93de26a910254ef", + "b469e4270fe34073890289b0aeca206e", + "c6a8151d21fd4bd7bf44e98734536e88", + "6d36a1f1c7454703a899656ae02f4168", + "a5132399be1e40998e6492c536901282", + "61fad31a93834cc88f233a7f6da121af", + "d8030ef670fb4e9d8d1510eebc443f1c", + "d2b343a815cc4d4f923249efdcef879e", + "b21e804d4383489eb60529c141f419f8", + "d7a97685fcff45edbfc4c4e86b64dce9", + "52a3043c0dec4844abef63728639d7aa", + "8acc6bf0c1fd492d82abb2fe942932f1", + "dc8abff41b1b42f38a1c8f851143dbdc", + "c649b4585e8e4384834b80c1d15e092c", + "1258202491504db19b18186f5051b97b", + "cd80389212a7495a88d5ea742bfca334", + "b052fd6d31cb4c3b877404280245fe09", + "4f2a35feaca549b9808b914c92ae5fe9", + "b90123f569174977b48deeabf961b0a1", + "fbcd6954a7754619aa8c280077b087cb", + "5803140078c44c3f84fb510cfcd8e866", + "7d0c1ce5f9854afab627747c1e78d8f1", + "6d1e44acd775451f969e29b4c3d5da20", + "51d09c1428ab470db4719c7e43c1ada2", + "9f596894527c40b2896333c7b6d2c79d", + "d1fc2d891fda4f2fa882c9cbd09f2164", + "be219bc6f11848c2802a5022fde5b1bf", + "b736a65868cc49969b31915c94ed7a1a", + "1699b0ccc8c14b95ab0b7b9d2e54c63f", + "35ebf54909894e2ba4c65815a771cb75", + "dd7d0f10db694ed1aea5580e9d5e964f", + "e54d069a0f4c4d5fb2b3aec94f19b88c", + "5be3dc01548c45c2959868b32f225093", + "c660fa94c7d04d9fbfd7ff4040412bbc", + "387f754f5437408289142ee215616b43", + "dc31e7a57a5440edbca3f90cf0da6ccc", + "68274fad39cb401d813e6e78aaa4740f", + "a1a6d69f3c294a7485b573d5982120b9", + "f079c93148824cacb472b522ba67da1b", + "ba2d84ed0e8a4732bb3f54b771c611fd", + "981040ad5099471f9e44bf290d45ef6f", + "8f019332a7844cc0a9d1fbd45562de2d", + "d7382fbdce804a109049d38accfcde12", + "d807c20c3e64421496cd4cba34ff292d", + "fd0887829e0048abbf67dbcee604ae66", + "dd696398cd5e4fac8df58f7e1c7c5fdc", + "47f736072db04c78942260a46adf5aea", + "e22bbab17a1942489ac1890876f3c70a", + "ea962db709014d56a380e64cf844cca2", + "bdcb3c8138c342f9bc750f6fe59a82aa", + "d0f1db3a304448748dfcb9ec58dd0635", + "c152bb8dae1f4a418d7312248648dfb3", + "b9a5d226da6d4a5488da1e32ce2b2e82", + "8cf4d25afcf4436fbfe335e393977fe6", + "419511a48638436c8cc0d671c059e437", + "925036c3cad345f79797488e138408eb", + "44ef1cc85aed44c88fdc9e1148fa9ef7", + "8a71cdbbb57c472c80ddb1b039ec727f", + "433b30e1478a497b833961e1e2cb4d5a", + "9701a264390f4ee0a5c6353f75a07275", + "215b4f54d7ba45abb329f1cf191df3b8", + "a4da84606c2d4a03b2d837c5162cd9d8", + "856b49936d4f4729896b2bfad40cbc43", + "368d790dd7d04c54a382d4fa312061ab", + "efd1517f79244069b90a81e746bf184f", + "7beb7ca3c3ee4b8aa50ad73e165eefbf", + "3d035e5b2b984f0da1f62cd8d0696f8c", + "ff82f55ff94e4948a566308bc211c2b5", + "577cc7116f07412f90140568819b7b75", + "b14a6c48596747f690185bf5862d17c2", + "b93020f8b46247019ce70c854023e0b0", + "bcfbfe5ebb024a98a33bed4fd80e768f", + "d46f7450a0564c82a8747be1fe51f8fa", + "5ca1910b18c14abaae243d29c9067d47", + "9e8f80d73307479682d5c830c196199f", + "51a6acd4d2734081882f2e37cc7ddc6b", + "b2dac98d38fd4165ae7d28665fd67b3d", + "adda65dea2f54861b77e518ab2117d4b", + "6d625be6ade44881a8f6100f09b333e9", + "06ad1a71c8a0417998f10b636dd9cd91", + "bdf6be73541c45d588360ace6fc7870c", + "a400db2a2f3f46519b6c8a422b8af0b2", + "41fc2202d34f42bab11bf0bcfd482add", + "c151201e7e724a33ab1d2d8ddac5b2a9", + "60748a3fc069456195e170945d2c5352", + "2c2391a2b4514d2881c3b3ec8648c5b4", + "62471169942648a9b4495eda0c70a55f", + "15a235b13c68404bac71c9e50545a711", + "2b7bec28af6b407f9f8560a3b5a6c84b", + "89f3f28a59d44e3b8ecb28951cc5d4b1", + "08f67e915bd94d06a963fb78f43b233d", + "39b2f1186d334f588f9b15c87d198b2d", + "490d4e5072fa43c58b3b6986b56fe5fd", + "8b6e6db8b24444f395ceba53979e22cc", + "e6b58c576ba249aba51a21b45f35e4d8", + "5abd31cd8f4b41d488f105f7baf18c3e", + "30378eeb1d064c41b1c194ab0bd17543", + "a4fd5d9b21444e72b7ee7c1d5d7ef2f7", + "5fb5d7d4b8d34189a71426e40abfc401", + "beeb09740d91458a95a3eacc3de64c9c", + "1909748b3b6d4d2787c7aa93e02cb7a2", + "def65225ded24775b7041ed2978dbeee", + "21e780899ec14dfe8d18272799c45ca6", + "5e34b06a9ea34690bbd1a6eb985c8699", + "9d2c517e52554769bdb69dc2a9b1b823", + "290c5dc1c2434280afc1b885ad332970", + "2175f1a46b7c45f089fc58b570350de5", + "2de499432804475f8f40687548fb63eb", + "758a42c106d2447897f800a4dbfa54fb", + "6c8ac0513e724d65b964ac810d0890d8", + "270f2342d7714c6baf72c1d4b0ce66d5", + "cea0fc4eb1654b02b4bb5a2c2c1c3afd", + "fda01e5fff3f4de49201dc6ab49bab68", + "8330d90195b143b580b9713fd4b7c927", + "bbb9ea224d34469ca2a32bf77dbfb309", + "f795810c5a044226aca068e071191bff", + "62c6883ce8a140889fe702ed11269596", + "aa94bfdd25e64d568b64efdde1f8e02d", + "ab03e8512dd54306a1d5fc616b273a1e", + "df805580e6cb46218a76f2a3d3188a47", + "12cf762493394fc79ecabf9f41ededc2", + "74e92d1ad4b34d82b018ab187c4e9e36", + "0cfe213d17014f868af9670cbce251a6", + "6eae20c34cc94a45aecc2f9bad71ad94", + "d4d8c52b87e44a548653403400339692", + "1404acc1ce084d0bb52bbd46d7888e44", + "efd49a29bbd641b189a90f9110add92f", + "094310cd34484656a0e77245b153bf99", + "b5d7e3e7b1a34ddd8ee35250216721df", + "2fa8891967fe4b89ae9588251b91f168", + "c02c15ea36d64418816cc75b216514ad", + "226720a7855d4baa9c78340d11ef7ce0", + "661b197428ec4f9ead3c026316016774", + "edc7479694e141caa14239028d6d6928", + "ec4405e0d3854a248e37c6cfb40c98ff", + "2799e52583a54f7a947f0a0846b62d51", + "3b83f024ce1b4071882f8a0ec0eded2c", + "24fc173d42bb4d7abb795304b7304fd8", + "66867ac4973245799ac1f1e82a149ce3", + "c5fbe71a84a5461787fae98964613022", + "966e68735f2b46fea111affe1f8f1320", + "f00738dd223d4d2d8cad9b99a42cea4f", + "6f5b186f60ed4244a4b32f52c8616ec9", + "4a6c90d511d04f6d8222422c215faa40", + "8d3e3122ac6249718a79f866a5cb3a33", + "d81d7cae6d1e42e3a6e049b4655c892e", + "1ae5937e5c78451ea5188a843b79c050", + "a64f887cfc7b4ca2836e760414bf2d85", + "e74806f4ca6149668b7c6f952a866fb7", + "bb7a24d59bdf456a98af83ae5406b71e", + "4c6cb2b32c0a492e8e290587a6b41ed6", + "a4460aacba4e4530a891dc74a0772000", + "7264b115983e4e9cb5b220141eaa7f42", + "6365d35fe5c84384908a955ba9f656ad", + "35fb8881c1bb485fba095d4db5d1bac4", + "ae6ade70349f4cee9cea3fff09b4aaf4", + "c03d7cd927e242999e4b7a6736bd22ef", + "2e612d85bcba41ea921cbf6df52b61fc", + "221aa135e2e7401fbe1ba38f42716378", + "c41a4793b7b74ae097faef919cf61a70", + "1a784c09a999423daede6f7eaa636dde", + "efdbd0835b1142029521f4e330a37c99", + "d4b4051e7b8e4fecb9ed79c08dab4d7e", + "791bdd214d724280aebb0379fd756495", + "0f20d82d76d94d8ab69b544ed2f9289f", + "0e5aaae852904e65bb4a53cc0b401319", + "0939a52140064a4ba8fb3768d3868dc2", + "b84b4be62bda46d2b9c2e1f13cf65638", + "b54c55fed6724027a33f716928b2c24f", + "62f0e021814945018275ecc4a70c0bbd", + "8ce8c907ebc54090b825584438af92be", + "0b6fa7ade10f45c99baadfa202b03797", + "2df0ce4207864d46aa69f95a9c4551f2", + "f61ece76017c47e185784df85f15bb8a", + "c886b1aad38b4a1090934425bd9fbc7e", + "18b60891f53b4d75b69f927305420295", + "b26443fcac344b80a9a9395bf9f8e027", + "15cf861836694d6bbeb3f28faeb621e8" + ] + }, + "id": "kYaaIJ3pD1jE", + "outputId": "351c5309-3545-4737-e91c-c0daf96229ab" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:llmware.models:ModelCatalog - load_model - fetching model - slim-sentiment-tool - from remote repository using pull_snapshot_from_hf - this may take a couple of minutes the first time.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step - \t3 - \tloading tool - sentiment\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_token.py:89: UserWarning: \n", + "The secret `HF_TOKEN` does not exist in your Colab secrets.\n", + "To authenticate with the Hugging Face Hub, create a token in your settings tab (https://huggingface.co/settings/tokens), set it as secret in your Google Colab and restart your session.\n", + "You will be able to reuse this secret in all of your notebooks.\n", + "Please note that authentication is recommended but still optional to access public models or datasets.\n", + " warnings.warn(\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "c50e606ff8ea41bfb369f6ccf368d238", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "agent.load_tool(\"sentiment\")\n", + "agent.load_tool(\"ner\")\n", + "\n", + "agent.load_tool_list([\"emotions\", \"topics\", \"intent\", \"tags\", \"ratings\", \"answer\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Pzi2QDb4ahJJ" + }, + "source": [ + "# Deploy tools and run analysis" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "U9AvagPPbGsL" + }, + "source": [ + "Assess \"soft skills\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 641, + "referenced_widgets": [ + "6e8c017741b94c4991efd41a2d049f5e", + "af1e6c37f2654cf2a492357174e52333", + "fe8dec363a754b089390b5c674c3abef", + "c05a5349a5134d10882e1a3fd23ef0ad", + "28b24e3acdd6465eb3c6bdd6b6fe9091", + "b452f05bdeb74b9c994b41e5c7dd66a1", + "e907a49e3eb84a0fb0521fdea9e8283f", + "0ff642fa936342e0b252a5ba462a568a", + "099a0e7a05c94a0d9a9208442d8b1ae6", + "fb3908d3a26a41da874bbd8f12491067", + "d23552be1c5f4a43802148f4dcfb70a8" + ] + }, + "id": "kxIk32w0ahUj", + "outputId": "2774cae4-8435-4d09-c41f-619ff41a6ea1" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step - \t11 - \texecuting function call - deploying - sentiment \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:llmware.models:update: LocalTokenizer - need to fetch tokenizer - tokenizer_tl.json\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "step - \t12 - \texecuting function call - getting response - sentiment\n", + "\t\t\t\t -- llm_response - {'sentiment': ['negative']}\n", + "\t\t\t\t -- output type - dict\n", + "\t\t\t\t -- usage - {'input': 107, 'output': 8, 'total': 115, 'metric': 'tokens', 'processing_time': 1.458883285522461, 'type': 'dict'}\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6e8c017741b94c4991efd41a2d049f5e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "tokenizer_tl.json: 0%| | 0.00/1.84M [00:00=1.24.53 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.34.153)\n", + "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.5)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.26.4)\n", + "Requirement already satisfied: pymongo>=4.7.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (4.8.0)\n", + "Requirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Requirement already satisfied: psycopg-binary==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: psycopg==3.1.17 in /usr/local/lib/python3.10/dist-packages (from llmware) (3.1.17)\n", + "Requirement already satisfied: pgvector==0.2.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.2.4)\n", + "Requirement already satisfied: colorama==0.4.6 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.4.6)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.2)\n", + "Requirement already satisfied: botocore<1.35.0,>=1.34.153 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.34.153)\n", + "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (1.0.1)\n", + "Requirement already satisfied: s3transfer<0.11.0,>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from boto3>=1.24.53->llmware) (0.10.2)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.15.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2024.6.1)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.13.1)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.3.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.60.0)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4.0)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Requirement already satisfied: dnspython<3.0.0,>=1.16.0 in /usr/local/lib/python3.10/dist-packages (from pymongo>=4.7.0->llmware) (2.6.1)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.153->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.153->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.44,>=0.43.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.43.0)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.7.4)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.153->boto3>=1.24.53->llmware) (1.16.0)\n" + ] + } + ], + "source": [ + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "EHlcnMoluzYE" + }, + "outputs": [], + "source": [ + "from llmware.web_services import YFinance\n", + "from llmware.models import ModelCatalog\n", + "from llmware.parsers import WikiParser\n", + "\n", + "from importlib import util\n", + "if not util.find_spec(\"yfinance\"):\n", + " print(\"\\nto run this example, you need to install yfinance first, e.g., pip3 install yfinance\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qP7mHzsMu5Tz", + "outputId": "d995017f-9de8-4f04-924e-47f9ab222326" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: yfinance in /usr/local/lib/python3.10/dist-packages (0.2.41)\n", + "Requirement already satisfied: pandas>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from yfinance) (2.1.4)\n", + "Requirement already satisfied: numpy>=1.16.5 in /usr/local/lib/python3.10/dist-packages (from yfinance) (1.26.4)\n", + "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.10/dist-packages (from yfinance) (2.31.0)\n", + "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.10/dist-packages (from yfinance) (0.0.11)\n", + "Requirement already satisfied: lxml>=4.9.1 in /usr/local/lib/python3.10/dist-packages (from yfinance) (4.9.4)\n", + "Requirement already satisfied: platformdirs>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from yfinance) (4.2.2)\n", + "Requirement already satisfied: pytz>=2022.5 in /usr/local/lib/python3.10/dist-packages (from yfinance) (2024.1)\n", + "Requirement already satisfied: frozendict>=2.3.4 in /usr/local/lib/python3.10/dist-packages (from yfinance) (2.4.4)\n", + "Requirement already satisfied: peewee>=3.16.2 in /usr/local/lib/python3.10/dist-packages (from yfinance) (3.17.6)\n", + "Requirement already satisfied: beautifulsoup4>=4.11.1 in /usr/local/lib/python3.10/dist-packages (from yfinance) (4.12.3)\n", + "Requirement already satisfied: html5lib>=1.1 in /usr/local/lib/python3.10/dist-packages (from yfinance) (1.1)\n", + "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4>=4.11.1->yfinance) (2.5)\n", + "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.10/dist-packages (from html5lib>=1.1->yfinance) (1.16.0)\n", + "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from html5lib>=1.1->yfinance) (0.5.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->yfinance) (2.8.2)\n", + "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->yfinance) (2024.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->yfinance) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->yfinance) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->yfinance) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->yfinance) (2024.7.4)\n" + ] + } + ], + "source": [ + "!pip3 install yfinance" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "H3vGToXDBJZV" + }, + "source": [ + "Sample text from CNBC earnings release" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "0FTLPYEqu7M9" + }, + "outputs": [], + "source": [ + "text=(\"BEAVERTON, Ore.--(BUSINESS WIRE)--NIKE, Inc. (NYSE:NKE) today reported fiscal 2024 financial results for its \"\n", + " \"third quarter ended February 29, 2024.) “We are making the necessary adjustments to drive NIKE’s next chapter \"\n", + " \"of growth Post this Third quarter revenues were slightly up on both a reported and currency-neutral basis* \"\n", + " \"at $12.4 billion NIKE Direct revenues were $5.4 billion, slightly up on a reported and currency-neutral basis \"\n", + " \"NIKE Brand Digital sales decreased 3 percent on a reported basis and 4 percent on a currency-neutral basis \"\n", + " \"Wholesale revenues were $6.6 billion, up 3 percent on a reported and currency-neutral basis Gross margin \"\n", + " \"increased 150 basis points to 44.8 percent, including a detriment of 50 basis points due to restructuring charges \"\n", + " \"Selling and administrative expense increased 7 percent to $4.2 billion, including $340 million of restructuring \"\n", + " \"charges Diluted earnings per share was $0.77, including $0.21 of restructuring charges. Excluding these \"\n", + " \"charges, Diluted earnings per share would have been $0.98* “We are making the necessary adjustments to \"\n", + " \"drive NIKE’s next chapter of growth,” said John Donahoe, President & CEO, NIKE, Inc. “We’re encouraged by \"\n", + " \"the progress we’ve seen, as we build a multiyear cycle of new innovation, sharpen our brand storytelling and \"\n", + " \"work with our wholesale partners to elevate and grow the marketplace.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XxFv9fIfBPgy" + }, + "source": [ + "Load three models" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 500, + "referenced_widgets": [ + "d5ad4b3b203f4767b25bad507118f37b", + "1d710e5d021b4b90866ba17457acc271", + "ead609aec8b64a8d8b593143895c58d5", + "824ced3f13504efc857c133265aa0a12", + "a4c26ee601d64d2cb8cb1715dceb63a8", + "f5751d5cb36a4609a4934302003d7a68", + "13f2541beed9440cae8c4d831dad2cd6", + "6b9b6eebc78245e187ea9f6092848ebc", + "0bf77ef8d23e4cabbc87fd9c6ee5698f", + "35abfee4ccf54bdc848a2ef946cba17b", + "ffcbf5df8725481292458f728c615bae", + "86ae4464ef7a430fa37ad6ee46dffa32", + "f5d720306dac4448947a66ab815b3d02", + "83d5e94862fc42e696c2568cda5d1eef", + "c52d98d532354139b8eb9dceeb8f9333", + "4c3add0649cd462fa6b4b2ea8eb5e64d", + "083718eb15c540ae8cf0660fc1d7f6d0", + "0c71163ee01346a191632c2c1051526f", + "4a283a014805476a8780aa02f9dfd6f3", + "89a1bd9c9a9a4244826e45515d637a57", + "b9cfa146e97246078ff93bd61ade016d", + "e073f28ac18a46dd8ee19802e21c5eb6", + "e11d7e4930dd48c7b91c534e5affef37", + "2bec5a335dbb45319f416299a45ac390", + "e86e70b34eb04eacaa7efe1b645efb9f", + "6406a4fde64846fab19eb0403f827b2c", + "962de7af9d1c4ee2bd5b0396db3ca69c", + "ebcf0ae5927f433a8792578535a932f3", + "feead54893bb4058b5bc527cd60420c3", + "1fd37612d73f4ba383760c0db5dccf0f", + "80f2febf4222405390a23cbd867a7327", + "200477073ea04606a4f54f86335389ad", + "334c317196a54d0886a7f12511da7d80", + "a99827d5810c48bbad3bc3470575e21b", + "4a5f584c420a4dc682239a74a9857a7a", + "48ffc30890e34e3ea02a865e5a5d0b7b", + "d84053fc20554fb4a8c652f9c91f24cf", + "034f434796ff4b6fa273391c6ffa43c4", + "753c5246fca04383b780002119f07ae3", + "e071f8a957344eedba097605b71bc1b6", + "696d62ec2f98422dadac40f0f374f82b", + "7503fe9473fb43b08e2b8d9e6436e172", + "190e38dc32eb4115bd78fe68b9a75e3d", + "ba7d5160e722458aa6dbc0c7aed8c655", + "683385b520dc4b209410150726d34160", + "127225b90b394415bc1b33aef7a37db5", + "bc600ce9071a4888a636953b9dae580b", + "fc5a4188e5164c6981cca1ea7a00a713", + "1c8bcffd3db5478f993d144f97332f07", + "95c92aefb9984a60a3ddcb1248cb6fe7", + "2541ca269b7e48e189172ae8378ed4a3", + "4d0c3eaa1d3c4e869b5153e61844efcf", + "ced0617011a04e0a8b16f18acc1b3885", + "2882f3ea04d8493eb8de628dcf45ee9d", + "01bfa09d36f84651ba9ba83613c2febf", + "a967699cf78944ce97d65f9c3410752d", + "44c2a1a743dc45b48de4b1a00b6422b9", + "cf685827d67f4868b952e034a3c21069", + "13b0b164fdac4402b87793e3281a4cf9", + "f033007f1a554a9882b5de9122704adb", + "ec80f91ed9ce49b38a048599bb577ae8", + "e3aef13f65c7478cb3cecd58bce96a00", + "53f0006dbf5a49e68b63701a4473fb80", + "4509aeb4b0454668be150ab21125fd17", + "fa87f1fc0d1e4082bfe70f3473d47d2b", + "ff0f116160074616ad4323eb1be13451", + "9c97f90e9e9c4de1bb9b141aaa1cfe32", + "90dd24bfd11d4b2a99d8b9bb623b7023", + "fb2a8f80beca43e8924a9d0637d335f4", + "0022e9bac57e447181d379539062b035", + "79f473434694473ea805160d960d5517", + "75df6494de6c4dcc9f7fbd1c52358d31", + "c1b659bdab274351b8f1e9ebab68339c", + "4afd1a6a7b8749d2bc92a5702604b879", + "a3e876c47a2048979f2d28afe542d9d5", + "60fb1639ad5b4bcf99e8636894548c30", + "05642f67fbab43c8b944461bb3f35abc", + "e425f1324e1549be83f5d4a688f4db4d", + "556631d052934cc6b389dd9b27bc7b9b", + "93edf1354c144fe4a28fae36c6c0a93b", + "4e0d3cc071814b5fbf5be3c7aca66a55", + "2a36be70673f47a2baf01435f4039095", + "393ffebe55924979994732a00d241b57", + "e9b6068410cd4cd6acb5b9fc22526583", + "937de54696bb4fd08d4256225f2f3c5b", + "d4957b8cc6054340a7affc5814e10a0f", + "3636d69c61784bbe95b0a0630c79abea", + "4d8009d95422426889cea2df14354a25", + "0917d90ca88b4c6a82c76894f8d8d680", + "05c3075d7f9b450398f8457c6e489173", + "f27bab30ccee4e97a9c64db1d14e694d", + "939ecb10ed8e440da5bcb14d7d530cd3", + "750e34de5f374af5bc689811836a8484", + "6620e5d309f643fdb7a2d8a7b6f58794", + "dc6965de814246c59392a3a0f8eef722", + "1b53f1c917bc4a6ca1c5584bd2e0020f", + "e3346c5511c5436fbe8ee525d18df7ff", + "a0ec74f3f7d7442ba82f403bef116227", + "5f17cb46d6e641a1a8003536686a3589", + "b8e2c5f9b0b2435baabe6c48bcefd880", + "eeebb53bb36d4a7c802b879ce0b2671d", + "5dc94e0aa10d4e86810681e898dee535", + "8192ab9ea9d149b6ac585345a57b6a32", + "a9e1564ed921418e949ed0de2ddabed4", + "a4645a6ce18b4835b21b0ea9e4bf01d1", + "09e1deea6fa54d40b96ced67fef3b619", + "b7d80e47907c476caca886bb9819fa7d", + "c1b3def10d7549be845ee74d6b25fcb6", + "a501e4157e834c2983f26d04d76b0119", + "190a668b4f7147c5a257f25a25f1a3e5" + ] + }, + "id": "jEvQNFN4vMTC", + "outputId": "81a1c690-3cca-4477-d02f-2289467cfd99" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:llmware.models:ModelCatalog - load_model - fetching model - slim-summary-tool - from remote repository using pull_snapshot_from_hf - this may take a couple of minutes the first time.\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "d5ad4b3b203f4767b25bad507118f37b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00 info -> ask questions#" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "-OzBcq-RFt59", + "outputId": "3ce0b140-ecde-44a7-94da-a241880c0b39" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Step 3 - use extracted company name to lookup in Wikipedia web service - and add background data\n", + "\n" + ] + } + ], + "source": [ + "print(\"\\nStep 3 - use extracted company name to lookup in Wikipedia web service - and add background data\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WV3Vi-iL0AL2", + "outputId": "5f5ecbf8-65ba-4e57-bc36-9a7f34a0fc59" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting wikipedia-api\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl.metadata (22 kB)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from wikipedia-api) (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (2024.7.4)\n", + "Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Installing collected packages: wikipedia-api\n", + "Successfully installed wikipedia-api-0.6.0\n" + ] + } + ], + "source": [ + "!pip install wikipedia-api" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "qAJ4lCpwFvs3", + "outputId": "fe5fe86e-250e-48a9-f137-216ad851f071" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-- calling summary model to summarize the first part of the Wikipedia article\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:llmware.models:update: function call output could not be automatically converted, but remediation was successful to type - list \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-- slim-summary - summary (5 points): {'llm_response': ['Nike Inc. was founded on January 25 1964 as ', 'by Bill Bowerman and Phil Knight', 'The company was officially named Nike Inc. on May 30 1971', 'Nike is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment', 'Nike sponsors many high-profile athletes and sports teams around the world', 'Nike is the most valuable brand among sports businesses'], 'usage': {'input': 380, 'output': 97, 'total': 477, 'metric': 'tokens', 'processing_time': 8.21936297416687, 'type': 'list', 'remediation': True}, 'logits': [[(13961, 1.0), (187, 0.0), (18256, 0.0), (1966, 0.0), (8801, 0.0), (78, 0.0), (14, 0.0), (67, 0.0), (15, 0.0), (1342, 0.0)], [(47, 0.661), (39, 0.171), (25310, 0.045), (8252, 0.02), (510, 0.019), (13652, 0.013), (2374, 0.012), (22423, 0.009), (32088, 0.006), (3, 0.004)], [(2804, 0.999), (6541, 0.0), (13972, 0.0), (587, 0.0), (366, 0.0), (460, 0.0), (1758, 0.0), (8678, 0.0), (6840, 0.0), (6327, 0.0)], [(13, 0.758), (369, 0.104), (310, 0.077), (3690, 0.031), (313, 0.007), (428, 0.005), (3053, 0.004), (11420, 0.003), (556, 0.001), (5550, 0.001)], [(3690, 0.982), (11420, 0.01), (310, 0.001), (253, 0.001), (247, 0.001), (271, 0.001), (50276, 0.001), (4232, 0.001), (369, 0.0), (11217, 0.0)], [(15, 0.889), (904, 0.048), (369, 0.023), (13, 0.013), (310, 0.01), (313, 0.004), (2464, 0.003), (428, 0.003), (1383, 0.002), (12567, 0.001)], [(369, 0.427), (310, 0.313), (313, 0.145), (11420, 0.022), (428, 0.022), (50276, 0.016), (5550, 0.012), (3053, 0.01), (50275, 0.003), (556, 0.003)], [(11420, 0.974), (4232, 0.007), (4447, 0.004), (11217, 0.003), (8927, 0.002), (3053, 0.002), (3562, 0.002), (253, 0.001), (10098, 0.001), (15335, 0.001)], [(327, 0.86), (347, 0.049), (275, 0.04), (407, 0.035), (4247, 0.009), (2552, 0.002), (13, 0.001), (3344, 0.001), (285, 0.001), (337, 0.0)], [(4247, 0.979), (3344, 0.017), (2552, 0.002), (4163, 0.001), (337, 0.0), (2030, 0.0), (4162, 0.0), (5080, 0.0), (3919, 0.0), (27, 0.0)], [(2030, 0.998), (13, 0.001), (2164, 0.0), (3436, 0.0), (15, 0.0), (1099, 0.0), (337, 0.0), (3435, 0.0), (50276, 0.0), (3349, 0.0)], [(13, 0.999), (15, 0.0), (1157, 0.0), (394, 0.0), (275, 0.0), (904, 0.0), (1383, 0.0), (347, 0.0), (14, 0.0), (209, 0.0)], [(17926, 0.989), (9354, 0.002), (29639, 0.002), (50276, 0.001), (6157, 0.001), (4059, 0.001), (12459, 0.001), (17601, 0.001), (1384, 0.001), (19949, 0.0)], [(13, 0.702), (1383, 0.213), (347, 0.05), (407, 0.023), (15, 0.005), (6038, 0.001), (40219, 0.001), (762, 0.001), (4117, 0.001), (28, 0.001)], [(347, 0.862), (407, 0.119), (285, 0.004), (1383, 0.003), (762, 0.002), (342, 0.002), (1925, 0.001), (4907, 0.001), (50276, 0.001), (275, 0.001)], [(346, 0.766), (10063, 0.127), (686, 0.091), (2634, 0.004), (773, 0.004), (50276, 0.002), (15078, 0.002), (20890, 0.001), (1383, 0.0), (50275, 0.0)], [(22036, 1.0), (35, 0.0), (47, 0.0), (5622, 0.0), (10063, 0.0), (510, 0.0), (5993, 0.0), (11863, 0.0), (15383, 0.0), (25310, 0.0)], [(31641, 0.999), (416, 0.0), (50276, 0.0), (28188, 0.0), (51, 0.0), (11879, 0.0), (37491, 0.0), (7121, 0.0), (1383, 0.0), (686, 0.0)], [(4006, 1.0), (21111, 0.0), (5568, 0.0), (3508, 0.0), (67, 0.0), (23798, 0.0), (4152, 0.0), (15801, 0.0), (4805, 0.0), (251, 0.0)], [(15000, 0.999), (21483, 0.001), (1383, 0.0), (995, 0.0), (2101, 0.0), (1608, 0.0), (9001, 0.0), (22344, 0.0), (50276, 0.0), (8, 0.0)], [(995, 0.501), (3, 0.466), (937, 0.01), (3446, 0.01), (449, 0.005), (3664, 0.004), (13, 0.002), (1383, 0.001), (9686, 0.0), (6624, 0.0)], [(407, 0.99), (1383, 0.004), (285, 0.003), (50276, 0.001), (327, 0.0), (342, 0.0), (11420, 0.0), (6038, 0.0), (8, 0.0), (275, 0.0)], [(7641, 0.998), (7252, 0.001), (50276, 0.0), (767, 0.0), (1383, 0.0), (33663, 0.0), (697, 0.0), (15217, 0.0), (50275, 0.0), (11731, 0.0)], [(10427, 0.997), (285, 0.001), (378, 0.0), (50276, 0.0), (1383, 0.0), (19466, 0.0), (15, 0.0), (686, 0.0), (7233, 0.0), (12143, 0.0)], [(8592, 1.0), (39968, 0.0), (36687, 0.0), (3766, 0.0), (35406, 0.0), (3796, 0.0), (693, 0.0), (6702, 0.0), (1808, 0.0), (257, 0.0)], [(285, 0.997), (1383, 0.001), (13, 0.001), (708, 0.001), (313, 0.0), (5550, 0.0), (50276, 0.0), (8, 0.0), (342, 0.0), (6038, 0.0)], [(6778, 0.98), (15887, 0.019), (1777, 0.0), (38881, 0.0), (50276, 0.0), (40864, 0.0), (1383, 0.0), (17250, 0.0), (13, 0.0), (50275, 0.0)], [(21570, 0.998), (10381, 0.001), (611, 0.001), (36051, 0.0), (15, 0.0), (1383, 0.0), (31292, 0.0), (26184, 0.0), (427, 0.0), (50276, 0.0)], [(1383, 0.865), (13, 0.057), (15, 0.039), (40219, 0.022), (2464, 0.007), (28, 0.006), (4117, 0.002), (6038, 0.001), (285, 0.001), (8, 0.0)], [(686, 0.952), (346, 0.029), (32589, 0.019), (34912, 0.0), (15078, 0.0), (2634, 0.0), (17169, 0.0), (30225, 0.0), (14412, 0.0), (50276, 0.0)], [(510, 0.383), (2374, 0.225), (47, 0.162), (1147, 0.139), (688, 0.031), (25310, 0.008), (21756, 0.004), (7130, 0.004), (3, 0.004), (38023, 0.003)], [(2567, 0.964), (1416, 0.013), (3565, 0.011), (806, 0.003), (13739, 0.002), (747, 0.001), (6487, 0.001), (1655, 0.001), (7138, 0.001), (33663, 0.0)], [(369, 0.661), (3395, 0.181), (15335, 0.074), (310, 0.047), (4391, 0.012), (556, 0.006), (19186, 0.002), (434, 0.002), (8671, 0.001), (4916, 0.001)], [(15335, 0.716), (11420, 0.082), (4907, 0.05), (19186, 0.032), (8927, 0.025), (11217, 0.011), (9288, 0.011), (27624, 0.01), (840, 0.009), (8523, 0.008)], [(4907, 0.866), (1929, 0.027), (1925, 0.025), (11217, 0.01), (4391, 0.009), (2489, 0.009), (27624, 0.008), (346, 0.004), (4232, 0.004), (11420, 0.003)], [(40864, 0.724), (346, 0.155), (347, 0.076), (41177, 0.018), (327, 0.008), (686, 0.007), (285, 0.006), (13, 0.003), (275, 0.001), (2634, 0.001)], [(13, 0.798), (327, 0.148), (275, 0.029), (3690, 0.016), (285, 0.002), (672, 0.002), (846, 0.001), (347, 0.001), (1383, 0.001), (432, 0.0)], [(3690, 0.976), (327, 0.018), (275, 0.002), (285, 0.001), (342, 0.0), (50276, 0.0), (347, 0.0), (846, 0.0), (533, 0.0), (672, 0.0)], [(15, 0.91), (904, 0.059), (327, 0.016), (13, 0.013), (275, 0.001), (2464, 0.001), (285, 0.0), (1383, 0.0), (40219, 0.0), (8634, 0.0)], [(327, 0.987), (275, 0.006), (285, 0.002), (2552, 0.001), (347, 0.001), (50276, 0.0), (846, 0.0), (342, 0.0), (407, 0.0), (672, 0.0)], [(2552, 0.997), (3919, 0.001), (4247, 0.001), (7216, 0.0), (5080, 0.0), (3978, 0.0), (4163, 0.0), (50276, 0.0), (4162, 0.0), (4397, 0.0)], [(1884, 0.996), (4562, 0.002), (3285, 0.001), (13, 0.0), (15, 0.0), (3349, 0.0), (1384, 0.0), (2030, 0.0), (1458, 0.0), (495, 0.0)], [(13, 0.998), (15, 0.001), (1383, 0.0), (1157, 0.0), (394, 0.0), (904, 0.0), (273, 0.0), (275, 0.0), (4117, 0.0), (16609, 0.0)], [(16609, 0.987), (25197, 0.002), (43425, 0.002), (10333, 0.001), (6585, 0.001), (15750, 0.001), (50276, 0.001), (10226, 0.001), (15621, 0.0), (4332, 0.0)], [(1383, 0.954), (28, 0.016), (13, 0.01), (15, 0.009), (40219, 0.008), (285, 0.001), (2464, 0.001), (342, 0.001), (407, 0.0), (4117, 0.0)], [(686, 0.956), (346, 0.029), (32589, 0.013), (15078, 0.001), (2634, 0.0), (14412, 0.0), (30225, 0.0), (17169, 0.0), (34912, 0.0), (2802, 0.0)], [(47, 0.516), (510, 0.264), (688, 0.072), (1909, 0.039), (1147, 0.028), (7130, 0.018), (14569, 0.01), (2374, 0.01), (7542, 0.009), (3378, 0.003)], [(2804, 1.0), (13972, 0.0), (460, 0.0), (18579, 0.0), (366, 0.0), (8678, 0.0), (1758, 0.0), (383, 0.0), (1479, 0.0), (413, 0.0)], [(310, 0.368), (10169, 0.194), (369, 0.187), (556, 0.055), (34423, 0.029), (434, 0.024), (4390, 0.017), (13, 0.013), (1024, 0.012), (17045, 0.008)], [(253, 0.898), (1024, 0.043), (4390, 0.019), (581, 0.011), (247, 0.008), (2783, 0.003), (1533, 0.003), (1929, 0.002), (21392, 0.002), (3063, 0.001)], [(1533, 0.918), (6253, 0.058), (954, 0.013), (11762, 0.002), (1655, 0.002), (4283, 0.002), (4156, 0.002), (5962, 0.001), (346, 0.0), (1852, 0.0)], [(434, 0.772), (4479, 0.186), (457, 0.022), (686, 0.012), (6253, 0.003), (8, 0.002), (17719, 0.001), (65, 0.001), (14, 0.0), (256, 0.0)], [(6253, 0.981), (4283, 0.013), (954, 0.005), (5962, 0.001), (1852, 0.0), (6657, 0.0), (2791, 0.0), (285, 0.0), (4585, 0.0), (1180, 0.0)], [(29622, 0.938), (24122, 0.034), (11662, 0.006), (3174, 0.005), (11716, 0.004), (9678, 0.003), (9001, 0.002), (14281, 0.001), (7138, 0.001), (24451, 0.001)], [(273, 0.989), (285, 0.004), (323, 0.003), (1383, 0.001), (13, 0.001), (604, 0.001), (275, 0.0), (390, 0.0), (50276, 0.0), (327, 0.0)], [(24122, 0.998), (9001, 0.0), (9678, 0.0), (39749, 0.0), (3939, 0.0), (11916, 0.0), (28802, 0.0), (3174, 0.0), (27799, 0.0), (622, 0.0)], [(12682, 0.637), (3174, 0.356), (622, 0.001), (9001, 0.001), (285, 0.001), (9678, 0.001), (22756, 0.001), (16037, 0.001), (3580, 0.0), (6500, 0.0)], [(285, 0.924), (1383, 0.042), (13, 0.03), (342, 0.002), (28, 0.001), (15, 0.001), (708, 0.0), (313, 0.0), (275, 0.0), (347, 0.0)], [(622, 0.965), (9001, 0.005), (310, 0.004), (14234, 0.004), (24122, 0.003), (9678, 0.003), (6500, 0.002), (643, 0.002), (556, 0.001), (28234, 0.001)], [(28091, 0.998), (609, 0.002), (274, 0.0), (1560, 0.0), (613, 0.0), (77, 0.0), (4420, 0.0), (83, 0.0), (620, 0.0), (3623, 0.0)], [(13, 0.347), (285, 0.315), (1383, 0.29), (342, 0.029), (28, 0.01), (15, 0.005), (347, 0.001), (275, 0.001), (4117, 0.001), (2403, 0.0)], [(285, 0.586), (342, 0.38), (347, 0.016), (247, 0.002), (2403, 0.002), (1690, 0.001), (2556, 0.001), (1907, 0.001), (534, 0.001), (275, 0.001)], [(247, 0.773), (2201, 0.114), (310, 0.045), (11662, 0.012), (581, 0.012), (671, 0.009), (253, 0.008), (556, 0.006), (352, 0.005), (16596, 0.003)], [(2201, 0.997), (11662, 0.002), (4283, 0.001), (1781, 0.0), (4156, 0.0), (11906, 0.0), (1534, 0.0), (2791, 0.0), (4471, 0.0), (2022, 0.0)], [(11662, 0.98), (9001, 0.009), (24320, 0.002), (16596, 0.001), (9678, 0.001), (14281, 0.001), (29622, 0.001), (10264, 0.001), (6500, 0.001), (275, 0.0)], [(273, 0.987), (1383, 0.005), (275, 0.002), (285, 0.001), (323, 0.001), (13, 0.001), (390, 0.0), (604, 0.0), (342, 0.0), (50276, 0.0)], [(9001, 0.978), (9678, 0.013), (643, 0.004), (6500, 0.002), (28802, 0.002), (84, 0.0), (24122, 0.0), (2710, 0.0), (50276, 0.0), (3580, 0.0)], [(6500, 0.996), (1383, 0.002), (285, 0.001), (14, 0.0), (3580, 0.0), (1298, 0.0), (28234, 0.0), (10229, 0.0), (13, 0.0), (622, 0.0)], [(1383, 0.697), (13, 0.168), (342, 0.086), (28, 0.021), (15, 0.015), (285, 0.008), (275, 0.001), (4117, 0.001), (313, 0.0), (326, 0.0)], [(686, 0.933), (346, 0.046), (32589, 0.018), (15078, 0.002), (34912, 0.0), (14412, 0.0), (2634, 0.0), (17169, 0.0), (30225, 0.0), (50276, 0.0)], [(47, 0.336), (688, 0.267), (510, 0.19), (1909, 0.053), (7130, 0.047), (1147, 0.036), (2374, 0.006), (3378, 0.005), (2214, 0.004), (27132, 0.004)], [(2804, 1.0), (13972, 0.0), (366, 0.0), (1479, 0.0), (18579, 0.0), (1758, 0.0), (466, 0.0), (8678, 0.0), (460, 0.0), (20592, 0.0)], [(34423, 0.247), (369, 0.183), (310, 0.175), (556, 0.125), (434, 0.06), (574, 0.039), (10169, 0.02), (671, 0.017), (17045, 0.012), (4390, 0.011)], [(1142, 0.943), (7418, 0.011), (1029, 0.01), (247, 0.008), (285, 0.007), (17812, 0.006), (2067, 0.003), (2201, 0.002), (4122, 0.002), (2710, 0.002)], [(1029, 0.921), (17812, 0.05), (973, 0.009), (11906, 0.003), (4122, 0.003), (8530, 0.003), (9001, 0.002), (4633, 0.002), (5702, 0.001), (1755, 0.001)], [(14, 0.987), (6222, 0.013), (14729, 0.0), (428, 0.0), (1268, 0.0), (23114, 0.0), (3045, 0.0), (48433, 0.0), (17045, 0.0), (19947, 0.0)], [(14729, 0.999), (34586, 0.0), (856, 0.0), (36505, 0.0), (6222, 0.0), (71, 0.0), (5251, 0.0), (15608, 0.0), (13382, 0.0), (4387, 0.0)], [(17812, 0.997), (9001, 0.001), (27799, 0.0), (6671, 0.0), (5702, 0.0), (285, 0.0), (13, 0.0), (24122, 0.0), (3394, 0.0), (9678, 0.0)], [(285, 0.918), (1383, 0.038), (13, 0.031), (1690, 0.004), (1475, 0.003), (11762, 0.001), (275, 0.001), (824, 0.001), (28, 0.001), (15, 0.0)], [(9001, 0.721), (6671, 0.247), (9678, 0.007), (8525, 0.004), (34423, 0.003), (556, 0.002), (17812, 0.002), (310, 0.002), (643, 0.001), (3394, 0.001)], [(6671, 0.994), (1383, 0.003), (13, 0.001), (285, 0.0), (2285, 0.0), (275, 0.0), (1690, 0.0), (5659, 0.0), (6038, 0.0), (3394, 0.0)], [(1475, 0.366), (1383, 0.275), (13, 0.232), (11762, 0.059), (285, 0.023), (15, 0.007), (275, 0.007), (1690, 0.006), (2439, 0.004), (6038, 0.004)], [(253, 1.0), (1533, 0.0), (13, 0.0), (1383, 0.0), (285, 0.0), (342, 0.0), (697, 0.0), (344, 0.0), (15, 0.0), (281, 0.0)], [(1533, 0.998), (20902, 0.002), (1383, 0.0), (3159, 0.0), (3645, 0.0), (11762, 0.0), (253, 0.0), (4156, 0.0), (2862, 0.0), (1475, 0.0)], [(1383, 0.585), (13, 0.331), (342, 0.03), (28, 0.017), (6038, 0.014), (285, 0.008), (15, 0.008), (1690, 0.003), (16445, 0.001), (762, 0.0)], [(686, 0.942), (346, 0.052), (15078, 0.002), (32589, 0.002), (2634, 0.0), (14412, 0.0), (17169, 0.0), (34912, 0.0), (20890, 0.0), (30225, 0.0)], [(47, 0.385), (688, 0.285), (510, 0.123), (1909, 0.084), (7130, 0.047), (1147, 0.019), (2374, 0.007), (2214, 0.006), (27132, 0.002), (3378, 0.002)], [(2804, 1.0), (13972, 0.0), (8678, 0.0), (366, 0.0), (1479, 0.0), (466, 0.0), (18579, 0.0), (20592, 0.0), (460, 0.0), (1758, 0.0)], [(310, 0.266), (369, 0.262), (556, 0.153), (17045, 0.063), (574, 0.045), (434, 0.04), (13, 0.025), (27532, 0.022), (6171, 0.013), (671, 0.013)], [(253, 0.377), (21392, 0.161), (581, 0.1), (17045, 0.088), (4390, 0.051), (247, 0.029), (671, 0.02), (4409, 0.018), (4122, 0.017), (7117, 0.011)], [(954, 0.713), (1533, 0.194), (6253, 0.01), (4585, 0.005), (854, 0.005), (2626, 0.004), (884, 0.004), (608, 0.002), (1852, 0.002), (1273, 0.002)], [(9865, 0.934), (14, 0.038), (4633, 0.01), (21392, 0.008), (4122, 0.004), (2791, 0.001), (8214, 0.001), (27576, 0.001), (7478, 0.0), (7561, 0.0)], [(7138, 0.97), (9001, 0.01), (4156, 0.005), (9678, 0.003), (41294, 0.002), (2567, 0.001), (2190, 0.001), (10567, 0.001), (285, 0.001), (1982, 0.001)], [(2190, 0.954), (275, 0.035), (13, 0.004), (273, 0.001), (11762, 0.001), (15995, 0.001), (323, 0.001), (285, 0.001), (342, 0.0), (1561, 0.0)], [(9001, 0.946), (9678, 0.028), (253, 0.014), (697, 0.003), (512, 0.001), (84, 0.001), (643, 0.001), (2201, 0.001), (4156, 0.0), (28802, 0.0)], [(9341, 0.842), (17739, 0.113), (2136, 0.018), (4413, 0.009), (14, 0.006), (1383, 0.003), (285, 0.002), (17057, 0.001), (6038, 0.001), (7138, 0.001)], [(6038, 0.432), (13, 0.407), (285, 0.045), (15, 0.033), (275, 0.022), (342, 0.019), (2464, 0.009), (1383, 0.009), (28, 0.004), (313, 0.003)], [(0, 0.997), (187, 0.003), (209, 0.0), (50276, 0.0), (50275, 0.0), (61, 0.0), (15, 0.0), (29, 0.0), (50274, 0.0), (870, 0.0)]], 'output_tokens': [5013, 47, 2804, 13, 3690, 15, 369, 11420, 327, 4247, 2030, 13, 17926, 13, 347, 346, 22036, 31641, 4006, 15000, 995, 407, 7641, 10427, 8592, 285, 6778, 21570, 1383, 686, 510, 2567, 369, 15335, 4907, 40864, 13, 3690, 15, 327, 2552, 1884, 13, 16609, 1383, 686, 47, 2804, 310, 253, 1533, 434, 6253, 29622, 273, 24122, 12682, 285, 622, 28091, 13, 285, 247, 2201, 11662, 273, 9001, 6500, 1383, 686, 47, 2804, 34423, 1142, 1029, 14, 14729, 17812, 285, 9001, 6671, 1475, 253, 1533, 1383, 686, 47, 2804, 310, 253, 954, 9865, 7138, 2190, 9001, 9341, 6038, 0]}\n", + "\n", + "-- calling extract model to get key piece of information from the Wikipedia article - company founding date\n", + "-- slim-extract - founding date: {'llm_response': {'founding_date': ['January 25, 1964']}, 'usage': {'input': 373, 'output': 13, 'total': 386, 'metric': 'tokens', 'processing_time': 2.4974167346954346, 'type': 'dict'}, 'logits': [[(13961, 0.999), (10816, 0.0), (73, 0.0), (1087, 0.0), (12080, 0.0), (18256, 0.0), (10984, 0.0), (8656, 0.0), (14077, 0.0), (1342, 0.0)], [(71, 0.722), (14541, 0.229), (31456, 0.033), (41607, 0.013), (19431, 0.002), (49053, 0.001), (14692, 0.0), (23569, 0.0), (2337, 0.0), (28983, 0.0)], [(13802, 1.0), (857, 0.0), (2261, 0.0), (23569, 0.0), (2995, 0.0), (702, 0.0), (9631, 0.0), (1573, 0.0), (662, 0.0), (23144, 0.0)], [(64, 1.0), (14, 0.0), (876, 0.0), (3333, 0.0), (18914, 0.0), (795, 0.0), (4414, 0.0), (5295, 0.0), (15, 0.0), (6038, 0.0)], [(2754, 1.0), (24275, 0.0), (2203, 0.0), (1201, 0.0), (1590, 0.0), (14151, 0.0), (7791, 0.0), (36377, 0.0), (2913, 0.0), (25610, 0.0)], [(5295, 0.998), (47499, 0.002), (64, 0.0), (6038, 0.0), (8, 0.0), (27, 0.0), (876, 0.0), (1381, 0.0), (22426, 0.0), (2464, 0.0)], [(14412, 0.992), (15640, 0.005), (544, 0.001), (8168, 0.001), (5013, 0.0), (686, 0.0), (46912, 0.0), (12196, 0.0), (27428, 0.0), (24345, 0.0)], [(22423, 0.939), (6791, 0.03), (24099, 0.006), (14060, 0.005), (23375, 0.002), (19591, 0.002), (18489, 0.001), (20443, 0.001), (1099, 0.001), (29639, 0.001)], [(2030, 0.973), (13, 0.015), (3436, 0.002), (2164, 0.001), (3435, 0.001), (3349, 0.001), (575, 0.001), (50276, 0.001), (337, 0.0), (3285, 0.0)], [(13, 0.998), (1157, 0.001), (15, 0.0), (1383, 0.0), (428, 0.0), (394, 0.0), (904, 0.0), (4117, 0.0), (2464, 0.0), (14, 0.0)], [(17926, 0.957), (9354, 0.011), (6157, 0.007), (29639, 0.003), (4059, 0.003), (12459, 0.003), (1384, 0.002), (16648, 0.001), (19213, 0.001), (8441, 0.001)], [(6038, 0.957), (2464, 0.038), (13, 0.001), (13995, 0.001), (1383, 0.0), (8, 0.0), (15, 0.0), (313, 0.0), (428, 0.0), (3401, 0.0)], [(94, 0.999), (0, 0.0), (748, 0.0), (62, 0.0), (13995, 0.0), (870, 0.0), (187, 0.0), (14412, 0.0), (2311, 0.0), (20499, 0.0)], [(0, 0.999), (187, 0.001), (209, 0.0), (50276, 0.0), (50275, 0.0), (50274, 0.0), (1852, 0.0), (50273, 0.0), (50269, 0.0), (50270, 0.0)]], 'output_tokens': [46912, 71, 13802, 64, 2754, 5295, 14412, 22423, 2030, 13, 17926, 6038, 94, 0]}\n", + "\n", + "-- calling extract model to get a short company business\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:llmware.models:update: function call output could not be automatically converted, but remediation was successful to type - dict \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-- slim-extract - response: {'llm_response': {'company_description': ['Nike Inc. is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.']}, 'usage': {'input': 373, 'output': 33, 'total': 406, 'metric': 'tokens', 'processing_time': 3.67295241355896, 'type': 'dict', 'remediation': True}, 'logits': [[(13961, 0.999), (10816, 0.0), (73, 0.0), (1087, 0.0), (12080, 0.0), (18256, 0.0), (10984, 0.0), (8656, 0.0), (14077, 0.0), (1342, 0.0)], [(25610, 1.0), (25590, 0.0), (32729, 0.0), (10008, 0.0), (10816, 0.0), (1590, 0.0), (14267, 0.0), (38885, 0.0), (46723, 0.0), (4674, 0.0)], [(64, 1.0), (876, 0.0), (5295, 0.0), (14, 0.0), (4414, 0.0), (3333, 0.0), (11185, 0.0), (15, 0.0), (27, 0.0), (47499, 0.0)], [(10008, 1.0), (3229, 0.0), (1590, 0.0), (12898, 0.0), (39930, 0.0), (28692, 0.0), (49027, 0.0), (20119, 0.0), (39990, 0.0), (14729, 0.0)], [(5295, 0.999), (47499, 0.001), (64, 0.0), (27, 0.0), (1381, 0.0), (6038, 0.0), (8, 0.0), (876, 0.0), (5456, 0.0), (5013, 0.0)], [(14412, 0.914), (15640, 0.082), (544, 0.003), (686, 0.0), (5550, 0.0), (346, 0.0), (12196, 0.0), (5013, 0.0), (8168, 0.0), (46912, 0.0)], [(47, 0.922), (510, 0.019), (13652, 0.016), (3, 0.008), (32088, 0.004), (46912, 0.003), (9819, 0.002), (7130, 0.002), (6267, 0.001), (8, 0.001)], [(2804, 0.999), (6541, 0.0), (460, 0.0), (6327, 0.0), (739, 0.0), (587, 0.0), (383, 0.0), (413, 0.0), (13972, 0.0), (15, 0.0)], [(13, 0.771), (310, 0.208), (3690, 0.008), (313, 0.004), (428, 0.002), (369, 0.001), (5550, 0.001), (2033, 0.001), (50276, 0.0), (556, 0.0)], [(3690, 0.986), (310, 0.005), (253, 0.002), (247, 0.001), (11420, 0.001), (11217, 0.001), (271, 0.001), (390, 0.0), (50276, 0.0), (21346, 0.0)], [(15, 0.873), (904, 0.075), (310, 0.019), (313, 0.008), (13, 0.008), (2464, 0.007), (14517, 0.004), (40219, 0.002), (12567, 0.001), (4880, 0.001)], [(310, 0.642), (313, 0.302), (5550, 0.021), (50276, 0.014), (369, 0.008), (428, 0.002), (390, 0.001), (50275, 0.001), (1680, 0.001), (12196, 0.001)], [(253, 0.444), (247, 0.271), (271, 0.26), (581, 0.013), (1533, 0.004), (2448, 0.001), (4390, 0.001), (2783, 0.001), (7117, 0.0), (2529, 0.0)], [(1533, 0.913), (954, 0.037), (6253, 0.036), (11762, 0.003), (4283, 0.002), (4156, 0.001), (530, 0.001), (2448, 0.0), (3645, 0.0), (5962, 0.0)], [(434, 0.715), (4479, 0.232), (457, 0.022), (686, 0.013), (6253, 0.006), (61, 0.005), (17719, 0.002), (8, 0.001), (65, 0.001), (6267, 0.001)], [(6253, 0.979), (954, 0.012), (4283, 0.007), (5962, 0.001), (6657, 0.0), (1852, 0.0), (1180, 0.0), (1682, 0.0), (1273, 0.0), (1755, 0.0)], [(29622, 0.922), (11662, 0.024), (24122, 0.017), (36631, 0.008), (11716, 0.006), (9001, 0.005), (622, 0.003), (24451, 0.003), (9678, 0.002), (2567, 0.001)], [(273, 0.993), (285, 0.002), (323, 0.001), (13, 0.001), (50276, 0.0), (16, 0.0), (604, 0.0), (390, 0.0), (61, 0.0), (313, 0.0)], [(24122, 0.994), (9001, 0.002), (9678, 0.001), (622, 0.001), (3939, 0.0), (27799, 0.0), (28802, 0.0), (3174, 0.0), (10567, 0.0), (11916, 0.0)], [(12682, 0.863), (3174, 0.117), (622, 0.004), (9001, 0.004), (3580, 0.003), (10229, 0.002), (285, 0.001), (8251, 0.001), (22756, 0.001), (6500, 0.001)], [(285, 0.957), (13, 0.033), (1383, 0.002), (2464, 0.002), (708, 0.001), (40219, 0.001), (15, 0.001), (6038, 0.001), (4880, 0.0), (8239, 0.0)], [(622, 0.982), (9001, 0.003), (14234, 0.002), (24122, 0.002), (310, 0.002), (7138, 0.001), (3580, 0.001), (643, 0.001), (10015, 0.001), (28234, 0.001)], [(28091, 0.998), (609, 0.001), (274, 0.0), (1560, 0.0), (83, 0.0), (4420, 0.0), (613, 0.0), (250, 0.0), (3623, 0.0), (11436, 0.0)], [(13, 0.442), (2464, 0.235), (285, 0.212), (15, 0.036), (40219, 0.024), (1383, 0.013), (6038, 0.013), (342, 0.009), (8239, 0.004), (4880, 0.004)], [(285, 0.745), (342, 0.199), (347, 0.029), (247, 0.009), (534, 0.001), (1481, 0.001), (1907, 0.001), (275, 0.001), (2556, 0.001), (310, 0.001)], [(247, 0.84), (310, 0.086), (581, 0.021), (253, 0.01), (671, 0.01), (2201, 0.009), (11662, 0.004), (556, 0.004), (352, 0.002), (271, 0.002)], [(2201, 0.99), (11662, 0.005), (4283, 0.002), (1781, 0.001), (1534, 0.0), (24320, 0.0), (4156, 0.0), (30263, 0.0), (1755, 0.0), (6657, 0.0)], [(11662, 0.967), (9001, 0.009), (24320, 0.006), (29622, 0.006), (16596, 0.003), (14281, 0.002), (1616, 0.001), (10264, 0.001), (11716, 0.001), (9678, 0.001)], [(273, 0.993), (275, 0.001), (323, 0.001), (13, 0.001), (285, 0.001), (15, 0.0), (50276, 0.0), (2464, 0.0), (6038, 0.0), (2033, 0.0)], [(9001, 0.969), (643, 0.012), (9678, 0.006), (6500, 0.004), (3580, 0.001), (50276, 0.001), (2710, 0.001), (28802, 0.001), (24122, 0.0), (273, 0.0)], [(6500, 0.993), (14, 0.001), (3580, 0.001), (285, 0.001), (1298, 0.001), (10229, 0.0), (30787, 0.0), (28234, 0.0), (2464, 0.0), (13, 0.0)], [(2464, 0.578), (13, 0.119), (15, 0.094), (40219, 0.093), (342, 0.049), (8239, 0.023), (1383, 0.013), (4880, 0.009), (285, 0.009), (6038, 0.005)], [(18095, 1.0), (12084, 0.0), (94, 0.0), (3291, 0.0), (2311, 0.0), (62, 0.0), (9102, 0.0), (599, 0.0), (44479, 0.0), (3117, 0.0)], [(0, 0.998), (187, 0.001), (209, 0.0), (50276, 0.0), (94, 0.0), (50275, 0.0), (50274, 0.0), (1852, 0.0), (50273, 0.0), (551, 0.0)]], 'output_tokens': [46912, 25610, 64, 10008, 5295, 14412, 47, 2804, 13, 3690, 15, 310, 253, 1533, 434, 6253, 29622, 273, 24122, 12682, 285, 622, 28091, 13, 285, 247, 2201, 11662, 273, 9001, 6500, 2464, 18095, 0]}\n", + "\n", + "-- asking a question directly to the Wikipedia article about the business\n", + "-- bling-answer model - response: {'llm_response': \" Nike, Inc. is the world's largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.\", 'usage': {'input': 375, 'output': 24, 'total': 399, 'metric': 'tokens', 'processing_time': 3.043423652648926}}\n", + "\n", + "-- asking a question about the origin of the company's name\n", + "-- bling-answer model - response: {'llm_response': ' Nike, the Greek goddess of victory.', 'usage': {'input': 376, 'output': 8, 'total': 384, 'metric': 'tokens', 'processing_time': 2.0494675636291504}}\n", + "\n", + "-- asking a question about the company's products\n", + "-- bling-answer model - response: {'llm_response': ' •Nike,\\r\\n•Nike Golf,\\r\\n•Nike Pro,\\r\\n•Nike+,\\r\\n•Nike Blazers,\\r\\n•Air Force 1,\\r\\n•Nike Dunk,\\r\\n•Air Max,\\r\\n•Foamposite,\\r\\n•Nike Skateboarding,\\r\\n•Nike CR7,\\r\\n•Bauer Hockey,\\r\\n•Cole Haan,\\r\\n•Umbro, and', 'usage': {'input': 371, 'output': 100, 'total': 471, 'metric': 'tokens', 'processing_time': 4.474163055419922}}\n", + "\n", + "\n", + "Step 4 - Completed Research - Summary Output\n", + "\n", + "research summary: {'stock_ticker': 'NYSE:NKE', 'company_name': 'NIKE, Inc.', 'total_revenues': '$12.4 billion', 'restructuring_charges': '$340 million', 'digital_growth': 'NIKE Brand Digital sales decreased 3% on a reported basis and 4% on a currency-neutral basis.', 'ceo_comment': 'NIKE is making the necessary adjustments to drive NIKEs next chapter of growth.', 'quarter_end_date': 'February 29, 2024', 'current_stock_price': 74.01, 'high_ltm': 123.39, 'low_ltm': 70.91, 'trailing_pe': 19.841824, 'forward_pe': 20.6156, 'volume': 10023093, 'market_cap': 110967635968, 'price_to_sales': 2.1605005, 'revenue_growth': -0.017, 'ebitda': 7598000128, 'gross_margin': 0.44685, 'currency': 'USD', 'sector': 'Consumer Cyclical', 'website': 'https://www.nike.com', 'industry': 'Footwear & Accessories', 'employees': 79400, 'officers': [('Mr. Mark G. Parker', 'Executive Chairman', 67, 6008438), ('Mr. John J. Donahoe II', 'President, CEO & Director', 62, 9946993), ('Mr. Matthew Friend', 'Executive VP & CFO', 45, 2290408), (\"Ms. Heidi O'Neill\", 'President of Consumer, Product & Brand', 57, 2299285), ('Mr. Craig Anthony Williams', 'President of Geographies & Marketplace', 53, 2263615), ('Mr. Philip H. Knight', 'Co-Founder & Chairman Emeritus', 85, 3069594), ('Ms. Johanna Nielsen', 'VP of Controlling & Principal Accounting Officer', 45, 'pay-NA'), ('Dr. Muge Erdirik Dogan', 'Chief Technology Officer', 'age-NA', 'pay-NA'), ('Mr. Paul Trussell C.F.A.', 'VP of Investor Relations & Strategic Finance', 'age-NA', 'pay-NA'), ('Ms. Ann M. Miller', 'Executive VP & Chief Legal Officer', 48, 'pay-NA')], 'summary': ['Nike Inc. was founded on January 25 1964 as ', 'by Bill Bowerman and Phil Knight', 'The company was officially named Nike Inc. on May 30 1971', 'Nike is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment', 'Nike sponsors many high-profile athletes and sports teams around the world', 'Nike is the most valuable brand among sports businesses'], 'founding_date': 'January 25, 1964', 'company_description': 'Nike Inc. is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.', 'business_overview': \" Nike, Inc. is the world's largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.\", 'origin_of_name': ' Nike, the Greek goddess of victory.', 'products': ' •Nike,\\r\\n•Nike Golf,\\r\\n•Nike Pro,\\r\\n•Nike+,\\r\\n•Nike Blazers,\\r\\n•Air Force 1,\\r\\n•Nike Dunk,\\r\\n•Air Max,\\r\\n•Foamposite,\\r\\n•Nike Skateboarding,\\r\\n•Nike CR7,\\r\\n•Bauer Hockey,\\r\\n•Cole Haan,\\r\\n•Umbro, and'}\n", + "\t\t -- 1 - \t - stock_ticker - NYSE:NKE \n", + "\t\t -- 2 - \t - company_name - NIKE, Inc. \n", + "\t\t -- 3 - \t - total_revenues - $12.4 billion \n", + "\t\t -- 4 - \t - restructuring_charges - $340 million \n", + "\t\t -- 5 - \t - digital_growth - NIKE Brand Digital sales decreased 3% on a reported basis and 4% on a currency-neutral basis.\n", + "\t\t -- 6 - \t - ceo_comment - NIKE is making the necessary adjustments to drive NIKEs next chapter of growth.\n", + "\t\t -- 7 - \t - quarter_end_date - February 29, 2024 \n", + "\t\t -- 8 - \t - current_stock_price - 74.01 \n", + "\t\t -- 9 - \t - high_ltm - 123.39 \n", + "\t\t -- 10 - \t - low_ltm - 70.91 \n", + "\t\t -- 11 - \t - trailing_pe - 19.841824 \n", + "\t\t -- 12 - \t - forward_pe - 20.6156 \n", + "\t\t -- 13 - \t - volume - 10023093 \n", + "\t\t -- 14 - \t - market_cap - 110967635968 \n", + "\t\t -- 15 - \t - price_to_sales - 2.1605005 \n", + "\t\t -- 16 - \t - revenue_growth - -0.017 \n", + "\t\t -- 17 - \t - ebitda - 7598000128 \n", + "\t\t -- 18 - \t - gross_margin - 0.44685 \n", + "\t\t -- 19 - \t - currency - USD \n", + "\t\t -- 20 - \t - sector - Consumer Cyclical \n", + "\t\t -- 21 - \t - website - https://www.nike.com \n", + "\t\t -- 22 - \t - industry - Footwear & Accessories \n", + "\t\t -- 23 - \t - employees - 79400 \n", + "\t\t -- 24 - \t - officers - [('Mr. Mark G. Parker', 'Executive Chairman', 67, 6008438), ('Mr. John J. Donahoe II', 'President, CEO & Director', 62, 9946993), ('Mr. Matthew Friend', 'Executive VP & CFO', 45, 2290408), (\"Ms. Heidi O'Neill\", 'President of Consumer, Product & Brand', 57, 2299285), ('Mr. Craig Anthony Williams', 'President of Geographies & Marketplace', 53, 2263615), ('Mr. Philip H. Knight', 'Co-Founder & Chairman Emeritus', 85, 3069594), ('Ms. Johanna Nielsen', 'VP of Controlling & Principal Accounting Officer', 45, 'pay-NA'), ('Dr. Muge Erdirik Dogan', 'Chief Technology Officer', 'age-NA', 'pay-NA'), ('Mr. Paul Trussell C.F.A.', 'VP of Investor Relations & Strategic Finance', 'age-NA', 'pay-NA'), ('Ms. Ann M. Miller', 'Executive VP & Chief Legal Officer', 48, 'pay-NA')]\n", + "\t\t -- 25 - \t - summary - ['Nike Inc. was founded on January 25 1964 as ', 'by Bill Bowerman and Phil Knight', 'The company was officially named Nike Inc. on May 30 1971', 'Nike is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment', 'Nike sponsors many high-profile athletes and sports teams around the world', 'Nike is the most valuable brand among sports businesses']\n", + "\t\t -- 26 - \t - founding_date - January 25, 1964 \n", + "\t\t -- 27 - \t - company_description - Nike Inc. is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.\n", + "\t\t -- 28 - \t - business_overview - Nike, Inc. is the world's largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.\n", + "\t\t -- 29 - \t - origin_of_name - Nike, the Greek goddess of victory. \n", + "\t\t -- 30 - \t - products - •Nike,•Nike Golf,•Nike Pro,•Nike+,•Nike Blazers,•Air Force 1,•Nike Dunk,•Air Max,•Foamposite,•Nike Skateboarding,•Nike CR7,•Bauer Hockey,•Cole Haan,•Umbro, and\n" + ] + } + ], + "source": [ + "if \"company_name\" in research_summary:\n", + "\n", + " company_name = research_summary[\"company_name\"]\n", + " output = WikiParser().add_wiki_topic(company_name, target_results=1)\n", + "\n", + " # get company summary\n", + " company_overview = \"\"\n", + " for i, blocks in enumerate(output[\"blocks\"]):\n", + " if i < 3:\n", + " company_overview += blocks[\"text\"]\n", + "\n", + " # call summary model to summarize\n", + " print(\"-- calling summary model to summarize the first part of the Wikipedia article\")\n", + " summary = model2.function_call(company_overview, params=[\"company history (5)\"])\n", + " print(\"-- slim-summary - summary (5 points): \", summary)\n", + "\n", + " research_summary.update({\"summary\": summary[\"llm_response\"]})\n", + "\n", + " # get founding date\n", + " print(\"\\n-- calling extract model to get key piece of information from the Wikipedia article - company founding date\")\n", + " response = model.function_call(company_overview, params=[\"founding date\"])\n", + " print(\"-- slim-extract - founding date: \", response)\n", + "\n", + " research_summary.update({\"founding_date\": response[\"llm_response\"][\"founding_date\"][0]})\n", + "\n", + " print(\"\\n-- calling extract model to get a short company business\")\n", + " response = model.function_call(company_overview, params=[\"company description\"])\n", + " print(\"-- slim-extract - response: \", response)\n", + " research_summary.update({\"company_description\": response[\"llm_response\"][\"company_description\"][0]})\n", + "\n", + " # ask other questions directly\n", + " print(\"\\n-- asking a question directly to the Wikipedia article about the business\")\n", + " response = model3.inference(\"What is an overview of company's business?\", add_context=company_overview)\n", + " print(\"-- bling-answer model - response: \", response)\n", + " research_summary.update({\"business_overview\": response[\"llm_response\"] })\n", + "\n", + " print(\"\\n-- asking a question about the origin of the company's name\")\n", + " response = model3.inference(\"What is the origin of the company's name?\", add_context=company_overview)\n", + " print(\"-- bling-answer model - response: \", response)\n", + " research_summary.update({\"origin_of_name\": response[\"llm_response\"]})\n", + "\n", + " print(\"\\n-- asking a question about the company's products\")\n", + " response = model3.inference(\"What are the product names\", add_context=company_overview)\n", + " print(\"-- bling-answer model - response: \", response)\n", + " research_summary.update({\"products\": response[\"llm_response\"]})\n", + "\n", + " print(\"\\n\\nStep 4 - Completed Research - Summary Output\\n\")\n", + " print(\"research summary: \", research_summary)\n", + "\n", + " item_counter = 1\n", + "\n", + " for keys, values in research_summary.items():\n", + " if isinstance(values, str):\n", + "\n", + " values = values.replace(\"\\n\", \"\")\n", + " values = values.replace(\"\\r\", \"\")\n", + " values = values.replace(\"\\t\", \"\")\n", + "\n", + " print(f\"\\t\\t -- {item_counter} - \\t - {keys.ljust(25)} - {str(values).ljust(40)}\")\n", + " item_counter += 1" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZdT5ydjwF7Yx", + "outputId": "8eec9cfe-08b3-438b-d739-5e971bd6e416" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'stock_ticker': 'NYSE:NKE', 'company_name': 'NIKE, Inc.', 'total_revenues': '$12.4 billion', 'restructuring_charges': '$340 million', 'digital_growth': 'NIKE Brand Digital sales decreased 3% on a reported basis and 4% on a currency-neutral basis.', 'ceo_comment': 'NIKE is making the necessary adjustments to drive NIKEs next chapter of growth.', 'quarter_end_date': 'February 29, 2024', 'current_stock_price': 74.01, 'high_ltm': 123.39, 'low_ltm': 70.91, 'trailing_pe': 19.841824, 'forward_pe': 20.6156, 'volume': 10023093, 'market_cap': 110967635968, 'price_to_sales': 2.1605005, 'revenue_growth': -0.017, 'ebitda': 7598000128, 'gross_margin': 0.44685, 'currency': 'USD', 'sector': 'Consumer Cyclical', 'website': 'https://www.nike.com', 'industry': 'Footwear & Accessories', 'employees': 79400, 'officers': [('Mr. Mark G. Parker', 'Executive Chairman', 67, 6008438), ('Mr. John J. Donahoe II', 'President, CEO & Director', 62, 9946993), ('Mr. Matthew Friend', 'Executive VP & CFO', 45, 2290408), (\"Ms. Heidi O'Neill\", 'President of Consumer, Product & Brand', 57, 2299285), ('Mr. Craig Anthony Williams', 'President of Geographies & Marketplace', 53, 2263615), ('Mr. Philip H. Knight', 'Co-Founder & Chairman Emeritus', 85, 3069594), ('Ms. Johanna Nielsen', 'VP of Controlling & Principal Accounting Officer', 45, 'pay-NA'), ('Dr. Muge Erdirik Dogan', 'Chief Technology Officer', 'age-NA', 'pay-NA'), ('Mr. Paul Trussell C.F.A.', 'VP of Investor Relations & Strategic Finance', 'age-NA', 'pay-NA'), ('Ms. Ann M. Miller', 'Executive VP & Chief Legal Officer', 48, 'pay-NA')], 'summary': ['Nike Inc. was founded on January 25 1964 as ', 'by Bill Bowerman and Phil Knight', 'The company was officially named Nike Inc. on May 30 1971', 'Nike is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment', 'Nike sponsors many high-profile athletes and sports teams around the world', 'Nike is the most valuable brand among sports businesses'], 'founding_date': 'January 25, 1964', 'company_description': 'Nike Inc. is the worlds largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.', 'business_overview': \" Nike, Inc. is the world's largest supplier of athletic shoes and apparel and a major manufacturer of sports equipment.\", 'origin_of_name': ' Nike, the Greek goddess of victory.', 'products': ' •Nike,\\r\\n•Nike Golf,\\r\\n•Nike Pro,\\r\\n•Nike+,\\r\\n•Nike Blazers,\\r\\n•Air Force 1,\\r\\n•Nike Dunk,\\r\\n•Air Max,\\r\\n•Foamposite,\\r\\n•Nike Skateboarding,\\r\\n•Nike CR7,\\r\\n•Bauer Hockey,\\r\\n•Cole Haan,\\r\\n•Umbro, and'}\n" + ] + } + ], + "source": [ + "print(research_summary)" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "0022e9bac57e447181d379539062b035": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_60fb1639ad5b4bcf99e8636894548c30", + "placeholder": "​", + "style": "IPY_MODEL_05642f67fbab43c8b944461bb3f35abc", + "value": " 247k/247k [00:00<00:00, 3.67MB/s]" + } + }, + "01bfa09d36f84651ba9ba83613c2febf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "034f434796ff4b6fa273391c6ffa43c4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "05642f67fbab43c8b944461bb3f35abc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "05c3075d7f9b450398f8457c6e489173": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6620e5d309f643fdb7a2d8a7b6f58794", + "placeholder": "​", + "style": "IPY_MODEL_dc6965de814246c59392a3a0f8eef722", + "value": "README.md: 100%" + } + }, + "083718eb15c540ae8cf0660fc1d7f6d0": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "0917d90ca88b4c6a82c76894f8d8d680": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_05c3075d7f9b450398f8457c6e489173", + "IPY_MODEL_f27bab30ccee4e97a9c64db1d14e694d", + "IPY_MODEL_939ecb10ed8e440da5bcb14d7d530cd3" + ], + "layout": "IPY_MODEL_750e34de5f374af5bc689811836a8484" + } + }, + "09e1deea6fa54d40b96ced67fef3b619": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "0bf77ef8d23e4cabbc87fd9c6ee5698f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "0c71163ee01346a191632c2c1051526f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "127225b90b394415bc1b33aef7a37db5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_95c92aefb9984a60a3ddcb1248cb6fe7", + "placeholder": "​", + "style": "IPY_MODEL_2541ca269b7e48e189172ae8378ed4a3", + "value": "slim-summarize.gguf: 100%" + } + }, + "13b0b164fdac4402b87793e3281a4cf9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_fa87f1fc0d1e4082bfe70f3473d47d2b", + "placeholder": "​", + "style": "IPY_MODEL_ff0f116160074616ad4323eb1be13451", + "value": " 4/4 [00:16<00:00,  6.26s/it]" + } + }, + "13f2541beed9440cae8c4d831dad2cd6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "190a668b4f7147c5a257f25a25f1a3e5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "190e38dc32eb4115bd78fe68b9a75e3d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1b53f1c917bc4a6ca1c5584bd2e0020f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1c8bcffd3db5478f993d144f97332f07": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "1d710e5d021b4b90866ba17457acc271": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_f5751d5cb36a4609a4934302003d7a68", + "placeholder": "​", + "style": "IPY_MODEL_13f2541beed9440cae8c4d831dad2cd6", + "value": "Fetching 4 files: 100%" + } + }, + "1fd37612d73f4ba383760c0db5dccf0f": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "200477073ea04606a4f54f86335389ad": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2541ca269b7e48e189172ae8378ed4a3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "2882f3ea04d8493eb8de628dcf45ee9d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2a36be70673f47a2baf01435f4039095": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "2bec5a335dbb45319f416299a45ac390": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ebcf0ae5927f433a8792578535a932f3", + "placeholder": "​", + "style": "IPY_MODEL_feead54893bb4058b5bc527cd60420c3", + "value": "README.md: 100%" + } + }, + "334c317196a54d0886a7f12511da7d80": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "35abfee4ccf54bdc848a2ef946cba17b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "3636d69c61784bbe95b0a0630c79abea": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "393ffebe55924979994732a00d241b57": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "44c2a1a743dc45b48de4b1a00b6422b9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_ec80f91ed9ce49b38a048599bb577ae8", + "placeholder": "​", + "style": "IPY_MODEL_e3aef13f65c7478cb3cecd58bce96a00", + "value": "Fetching 4 files: 100%" + } + }, + "4509aeb4b0454668be150ab21125fd17": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "48ffc30890e34e3ea02a865e5a5d0b7b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_696d62ec2f98422dadac40f0f374f82b", + "max": 38775, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_7503fe9473fb43b08e2b8d9e6436e172", + "value": 38775 + } + }, + "4a283a014805476a8780aa02f9dfd6f3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4a5f584c420a4dc682239a74a9857a7a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_753c5246fca04383b780002119f07ae3", + "placeholder": "​", + "style": "IPY_MODEL_e071f8a957344eedba097605b71bc1b6", + "value": "config.json: 100%" + } + }, + "4afd1a6a7b8749d2bc92a5702604b879": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4c3add0649cd462fa6b4b2ea8eb5e64d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4d0c3eaa1d3c4e869b5153e61844efcf": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "4d8009d95422426889cea2df14354a25": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4e0d3cc071814b5fbf5be3c7aca66a55": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_3636d69c61784bbe95b0a0630c79abea", + "placeholder": "​", + "style": "IPY_MODEL_4d8009d95422426889cea2df14354a25", + "value": " 1.57k/1.57k [00:00<00:00, 37.6kB/s]" + } + }, + "53f0006dbf5a49e68b63701a4473fb80": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "556631d052934cc6b389dd9b27bc7b9b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_393ffebe55924979994732a00d241b57", + "placeholder": "​", + "style": "IPY_MODEL_e9b6068410cd4cd6acb5b9fc22526583", + "value": ".gitattributes: 100%" + } + }, + "5dc94e0aa10d4e86810681e898dee535": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b7d80e47907c476caca886bb9819fa7d", + "max": 1708595168, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c1b3def10d7549be845ee74d6b25fcb6", + "value": 1708595168 + } + }, + "5f17cb46d6e641a1a8003536686a3589": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "60fb1639ad5b4bcf99e8636894548c30": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6406a4fde64846fab19eb0403f827b2c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_200477073ea04606a4f54f86335389ad", + "placeholder": "​", + "style": "IPY_MODEL_334c317196a54d0886a7f12511da7d80", + "value": " 1.96k/1.96k [00:00<00:00, 68.6kB/s]" + } + }, + "6620e5d309f643fdb7a2d8a7b6f58794": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "683385b520dc4b209410150726d34160": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_127225b90b394415bc1b33aef7a37db5", + "IPY_MODEL_bc600ce9071a4888a636953b9dae580b", + "IPY_MODEL_fc5a4188e5164c6981cca1ea7a00a713" + ], + "layout": "IPY_MODEL_1c8bcffd3db5478f993d144f97332f07" + } + }, + "696d62ec2f98422dadac40f0f374f82b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "6b9b6eebc78245e187ea9f6092848ebc": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7503fe9473fb43b08e2b8d9e6436e172": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "750e34de5f374af5bc689811836a8484": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "753c5246fca04383b780002119f07ae3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "75df6494de6c4dcc9f7fbd1c52358d31": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "79f473434694473ea805160d960d5517": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "80f2febf4222405390a23cbd867a7327": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "8192ab9ea9d149b6ac585345a57b6a32": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a501e4157e834c2983f26d04d76b0119", + "placeholder": "​", + "style": "IPY_MODEL_190a668b4f7147c5a257f25a25f1a3e5", + "value": " 1.71G/1.71G [00:16<00:00, 132MB/s]" + } + }, + "824ced3f13504efc857c133265aa0a12": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_35abfee4ccf54bdc848a2ef946cba17b", + "placeholder": "​", + "style": "IPY_MODEL_ffcbf5df8725481292458f728c615bae", + "value": " 4/4 [00:09<00:00,  3.08s/it]" + } + }, + "83d5e94862fc42e696c2568cda5d1eef": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4a283a014805476a8780aa02f9dfd6f3", + "max": 1630, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_89a1bd9c9a9a4244826e45515d637a57", + "value": 1630 + } + }, + "86ae4464ef7a430fa37ad6ee46dffa32": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f5d720306dac4448947a66ab815b3d02", + "IPY_MODEL_83d5e94862fc42e696c2568cda5d1eef", + "IPY_MODEL_c52d98d532354139b8eb9dceeb8f9333" + ], + "layout": "IPY_MODEL_4c3add0649cd462fa6b4b2ea8eb5e64d" + } + }, + "89a1bd9c9a9a4244826e45515d637a57": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "90dd24bfd11d4b2a99d8b9bb623b7023": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_75df6494de6c4dcc9f7fbd1c52358d31", + "placeholder": "​", + "style": "IPY_MODEL_c1b659bdab274351b8f1e9ebab68339c", + "value": "config.json: 100%" + } + }, + "937de54696bb4fd08d4256225f2f3c5b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "939ecb10ed8e440da5bcb14d7d530cd3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a0ec74f3f7d7442ba82f403bef116227", + "placeholder": "​", + "style": "IPY_MODEL_5f17cb46d6e641a1a8003536686a3589", + "value": " 1.58k/1.58k [00:00<00:00, 110kB/s]" + } + }, + "93edf1354c144fe4a28fae36c6c0a93b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_937de54696bb4fd08d4256225f2f3c5b", + "max": 1575, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_d4957b8cc6054340a7affc5814e10a0f", + "value": 1575 + } + }, + "95c92aefb9984a60a3ddcb1248cb6fe7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "962de7af9d1c4ee2bd5b0396db3ca69c": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9c97f90e9e9c4de1bb9b141aaa1cfe32": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_90dd24bfd11d4b2a99d8b9bb623b7023", + "IPY_MODEL_fb2a8f80beca43e8924a9d0637d335f4", + "IPY_MODEL_0022e9bac57e447181d379539062b035" + ], + "layout": "IPY_MODEL_79f473434694473ea805160d960d5517" + } + }, + "a0ec74f3f7d7442ba82f403bef116227": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a3e876c47a2048979f2d28afe542d9d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "a4645a6ce18b4835b21b0ea9e4bf01d1": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a4c26ee601d64d2cb8cb1715dceb63a8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a501e4157e834c2983f26d04d76b0119": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a967699cf78944ce97d65f9c3410752d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_44c2a1a743dc45b48de4b1a00b6422b9", + "IPY_MODEL_cf685827d67f4868b952e034a3c21069", + "IPY_MODEL_13b0b164fdac4402b87793e3281a4cf9" + ], + "layout": "IPY_MODEL_f033007f1a554a9882b5de9122704adb" + } + }, + "a99827d5810c48bbad3bc3470575e21b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_4a5f584c420a4dc682239a74a9857a7a", + "IPY_MODEL_48ffc30890e34e3ea02a865e5a5d0b7b", + "IPY_MODEL_d84053fc20554fb4a8c652f9c91f24cf" + ], + "layout": "IPY_MODEL_034f434796ff4b6fa273391c6ffa43c4" + } + }, + "a9e1564ed921418e949ed0de2ddabed4": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b7d80e47907c476caca886bb9819fa7d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "b8e2c5f9b0b2435baabe6c48bcefd880": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_eeebb53bb36d4a7c802b879ce0b2671d", + "IPY_MODEL_5dc94e0aa10d4e86810681e898dee535", + "IPY_MODEL_8192ab9ea9d149b6ac585345a57b6a32" + ], + "layout": "IPY_MODEL_a9e1564ed921418e949ed0de2ddabed4" + } + }, + "b9cfa146e97246078ff93bd61ade016d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ba7d5160e722458aa6dbc0c7aed8c655": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "bc600ce9071a4888a636953b9dae580b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4d0c3eaa1d3c4e869b5153e61844efcf", + "max": 1708595168, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ced0617011a04e0a8b16f18acc1b3885", + "value": 1708595168 + } + }, + "c1b3def10d7549be845ee74d6b25fcb6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "c1b659bdab274351b8f1e9ebab68339c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c52d98d532354139b8eb9dceeb8f9333": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b9cfa146e97246078ff93bd61ade016d", + "placeholder": "​", + "style": "IPY_MODEL_e073f28ac18a46dd8ee19802e21c5eb6", + "value": " 1.63k/1.63k [00:00<00:00, 99.5kB/s]" + } + }, + "ced0617011a04e0a8b16f18acc1b3885": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cf685827d67f4868b952e034a3c21069": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_53f0006dbf5a49e68b63701a4473fb80", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_4509aeb4b0454668be150ab21125fd17", + "value": 4 + } + }, + "d4957b8cc6054340a7affc5814e10a0f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "d5ad4b3b203f4767b25bad507118f37b": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_1d710e5d021b4b90866ba17457acc271", + "IPY_MODEL_ead609aec8b64a8d8b593143895c58d5", + "IPY_MODEL_824ced3f13504efc857c133265aa0a12" + ], + "layout": "IPY_MODEL_a4c26ee601d64d2cb8cb1715dceb63a8" + } + }, + "d84053fc20554fb4a8c652f9c91f24cf": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_190e38dc32eb4115bd78fe68b9a75e3d", + "placeholder": "​", + "style": "IPY_MODEL_ba7d5160e722458aa6dbc0c7aed8c655", + "value": " 38.8k/38.8k [00:00<00:00, 622kB/s]" + } + }, + "dc6965de814246c59392a3a0f8eef722": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e071f8a957344eedba097605b71bc1b6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e073f28ac18a46dd8ee19802e21c5eb6": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e11d7e4930dd48c7b91c534e5affef37": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_2bec5a335dbb45319f416299a45ac390", + "IPY_MODEL_e86e70b34eb04eacaa7efe1b645efb9f", + "IPY_MODEL_6406a4fde64846fab19eb0403f827b2c" + ], + "layout": "IPY_MODEL_962de7af9d1c4ee2bd5b0396db3ca69c" + } + }, + "e3346c5511c5436fbe8ee525d18df7ff": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "e3aef13f65c7478cb3cecd58bce96a00": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "e425f1324e1549be83f5d4a688f4db4d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_556631d052934cc6b389dd9b27bc7b9b", + "IPY_MODEL_93edf1354c144fe4a28fae36c6c0a93b", + "IPY_MODEL_4e0d3cc071814b5fbf5be3c7aca66a55" + ], + "layout": "IPY_MODEL_2a36be70673f47a2baf01435f4039095" + } + }, + "e86e70b34eb04eacaa7efe1b645efb9f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1fd37612d73f4ba383760c0db5dccf0f", + "max": 1964, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_80f2febf4222405390a23cbd867a7327", + "value": 1964 + } + }, + "e9b6068410cd4cd6acb5b9fc22526583": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ead609aec8b64a8d8b593143895c58d5": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_6b9b6eebc78245e187ea9f6092848ebc", + "max": 4, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_0bf77ef8d23e4cabbc87fd9c6ee5698f", + "value": 4 + } + }, + "ebcf0ae5927f433a8792578535a932f3": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ec80f91ed9ce49b38a048599bb577ae8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "eeebb53bb36d4a7c802b879ce0b2671d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a4645a6ce18b4835b21b0ea9e4bf01d1", + "placeholder": "​", + "style": "IPY_MODEL_09e1deea6fa54d40b96ced67fef3b619", + "value": "bling-stablelm.gguf: 100%" + } + }, + "f033007f1a554a9882b5de9122704adb": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f27bab30ccee4e97a9c64db1d14e694d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_1b53f1c917bc4a6ca1c5584bd2e0020f", + "max": 1581, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e3346c5511c5436fbe8ee525d18df7ff", + "value": 1581 + } + }, + "f5751d5cb36a4609a4934302003d7a68": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "f5d720306dac4448947a66ab815b3d02": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_083718eb15c540ae8cf0660fc1d7f6d0", + "placeholder": "​", + "style": "IPY_MODEL_0c71163ee01346a191632c2c1051526f", + "value": ".gitattributes: 100%" + } + }, + "fa87f1fc0d1e4082bfe70f3473d47d2b": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "fb2a8f80beca43e8924a9d0637d335f4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4afd1a6a7b8749d2bc92a5702604b879", + "max": 246836, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_a3e876c47a2048979f2d28afe542d9d5", + "value": 246836 + } + }, + "fc5a4188e5164c6981cca1ea7a00a713": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_2882f3ea04d8493eb8de628dcf45ee9d", + "placeholder": "​", + "style": "IPY_MODEL_01bfa09d36f84651ba9ba83613c2febf", + "value": " 1.71G/1.71G [00:08<00:00, 237MB/s]" + } + }, + "feead54893bb4058b5bc527cd60420c3": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ff0f116160074616ad4323eb1be13451": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "ffcbf5df8725481292458f728c615bae": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example1.py b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example1.py new file mode 100644 index 0000000..b3ce639 --- /dev/null +++ b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example1.py @@ -0,0 +1,212 @@ +import marimo + +__generated_with = "0.5.2" +app = marimo.App() + + +@app.cell +def __(): + import marimo as mo + return mo, + + +@app.cell +def __(mo): + mo.md( + """ Fast Start Example #1 - Library - converting document files into an indexed knowledge collection. + + In this example, we will illustrate a basic recipe for completing the following steps: + + 1. Create a library as a organizing construct for your knowledge-base + 2. Download sample files for a Fast Start - easy to 'swap out' and replace with your own files + 3. Use library.add_files method to automatically parse, text chunk and index the documents + 4. Run a basic text query against your new Library + """) + return + + +@app.cell +def __(mo): + mo.md(f"\n**Step 0** - Import libraries, set database, initialize libraries, create sample folder") + return + + +@app.cell +def __(): + import os + from llmware.library import Library + from llmware.retrieval import Query + from llmware.setup import Setup + from llmware.configs import LLMWareConfig + return LLMWareConfig, Library, Query, Setup, os + + +@app.cell +def __(LLMWareConfig): + # optional - set the active DB to be used - by default, it is "mongo" + # if you are just getting started, and have not installed a separate db, select "sqlite" + LLMWareConfig().set_active_db("sqlite") + + library_name = "example1_library" + + # this is a list of document folders that will be pulled down by calling Setup() + sample_folders = ["Agreements", "Invoices", "UN-Resolutions-500", "SmallLibrary", "FinDocs", "AgreementsLarge"] + + # select one of the sample folders + selected_folder = sample_folders[0] + return library_name, sample_folders, selected_folder + + +@app.cell +def __(library_name, mo): + mo.md(f"\n**Step 1** - creating library: `{library_name}`") + return + + +@app.cell +def __(Library, library_name): + # create new library + library = Library().create_new_library(library_name) + return library, + + +@app.cell +def __(Setup): + # load the llmware sample files + # -- note: if you have used this example previously, UN-Resolutions-500 is new path + # -- to pull updated sample files, set: 'over_write=True' + sample_files_path = Setup().load_sample_files(over_write=False) + return sample_files_path, + + +@app.cell +def __(mo, sample_files_path): + mo.md(f"\n**Step 2** - loading the llmware sample files and saving at: {sample_files_path}`") + return + + +@app.cell +def __(os, sample_files_path, selected_folder): + sample_folder = selected_folder + # note: to replace with your own documents, just point to a local folder path that has the documents + ingestion_folder_path = os.path.join(sample_files_path, sample_folder) + return ingestion_folder_path, sample_folder + + +@app.cell +def __(ingestion_folder_path, mo): + mo.md(f"\n**Step 3** - parsing and indexing files from: {ingestion_folder_path}`") + return + + +@app.cell +def __(ingestion_folder_path, library): + # add files is the key ingestion method - parses, text chunks and indexes all files in folder + # --will automatically route to correct parser based on file extension + # --supported file extensions: .pdf, .pptx, .docx, .xlsx, .csv, .md, .txt, .json, .wav, and .zip, .jpg, .png + parsing_output = library.add_files(ingestion_folder_path) + return parsing_output, + + +@app.cell +def __(mo): + mo.md(f"\n**Step 4** - completed parsing") + return + + +@app.cell +def __(mo, parsing_output): + mo.md(f"{parsing_output}`") + return + + +@app.cell +def __(library): + # check the updated library card + updated_library_card = library.get_library_card() + doc_count = updated_library_card["documents"] + block_count = updated_library_card["blocks"] + return block_count, doc_count, updated_library_card + + +@app.cell +def __(mo): + mo.md(f"\n **Step 5** - updated library card") + return + + +@app.cell +def __(block_count, doc_count, mo, updated_library_card): + mo.md(f"documents - {doc_count}, blocks - {block_count}, updated library card - {updated_library_card}") + return + + +@app.cell +def __(library): + # check the main folder structure created for the library - check /images to find extracted images + library_path = library.library_main_path + return library_path, + + +@app.cell +def __(library_path, mo): + mo.md(f"**Step 6** - library artifacts - including extracted images - saved at folder path - {library_path}") + return + + +@app.cell +def __(): + # use .add_files as many times as needed to build up your library, and/or create different libraries for + # different knowledge bases + # now, your library is ready to go and you can start to use the library for running queries + # if you are using the "Agreements" library, then a good easy 'hello world' query is "base salary" + # if you are using one of the other sample folders (or your own), then you should adjust the query + # queries are always created the same way, e.g., instantiate a Query object, and pass a library object + # --within the Query class, there are a variety of useful methods to run different types of queries + test_query = "base salary" + return test_query, + + +@app.cell +def __(mo, test_query): + mo.md(f"**Step 7** - running a test query - {test_query}\n") + return + + +@app.cell +def __(Query, library, test_query): + query_results = Query(library).text_query(test_query, result_count=10) + return query_results, + + +@app.cell +def __(query_results): + for i, result in enumerate(query_results): + + # note: each result is a dictionary with a wide range of useful keys + # -- we would encourage you to take some time to review each of the keys and the type of metadata available + + # here are a few useful attributes + text = result["text"] + file_source = result["file_source"] + page_number = result["page_num"] + doc_id = result["doc_ID"] + block_id = result["block_ID"] + matches = result["matches"] + + # -- if print to console is too verbose, then pick just a few keys for print + print("query results: ", i, result) + return ( + block_id, + doc_id, + file_source, + i, + matches, + page_number, + result, + text, + ) + + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example2.py b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example2.py new file mode 100644 index 0000000..0000b77 --- /dev/null +++ b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example2.py @@ -0,0 +1,230 @@ +import marimo + +__generated_with = "0.5.2" +app = marimo.App() + + +@app.cell +def __(): + import marimo as mo + return mo, + + +@app.cell +def __(mo): + mo.md("""" Fast Start Example #2 - Embeddings - applying an embedding model to enable natural language queries + + In this example, we will show the basic recipe for creating embeddings on a library: + + 1. Create a sample library (see Example #1 for more details) + 2 Select an embedding model + 3. Select a vector db + 4. Install the embeddings + 5. Run a semantic test query + + For purpose of this 'fast start', we will use a no-install option of 'chromadb' and 'sqlite' + + Note: we have updated the no-install vector db option to 'chromadb' from 'faiss' starting in + llmware>=0.2.12, due to better support on Python 3.12 + + Note: you may need to install chromadb's python driver: `pip3 install chromadb` + + -- This same basic recipe will work with any of the vector db and collection db by simply changing the name + + """) + return + + +@app.cell +def __(mo): + mo.md(f"\n**Step 0** - Import libraries, set database, initialize libraries, create sample folder, select embedding model") + return + + +@app.cell +def __(): + import os + from llmware.library import Library + from llmware.retrieval import Query + from llmware.setup import Setup + from llmware.status import Status + from llmware.models import ModelCatalog + from llmware.configs import LLMWareConfig + + from importlib import util + if not util.find_spec("chromadb"): + print("\nto run this example with chromadb, you need to install the chromadb python sdk: pip3 install chromadb") + return ( + LLMWareConfig, + Library, + ModelCatalog, + Query, + Setup, + Status, + os, + util, + ) + + +@app.cell +def __(LLMWareConfig): + LLMWareConfig().set_active_db("sqlite") + LLMWareConfig().set_vector_db("chromadb") + return + + +@app.cell +def __(ModelCatalog): + embedding_models = ModelCatalog().list_embedding_models() + embedding_model = "mini-lm-sbert" + return embedding_model, embedding_models + + +@app.cell +def __(): + library_name = "example2_library" + return library_name, + + +@app.cell +def __(mo): + mo.md(" \n**1** - Create library") + return + + +@app.cell +def __(mo): + mo.md(" \n**Step 1** - Create library which is the main 'organizing construct' in llmware") + return + + +@app.cell +def __(Library, library_name): + print ("\nupdate: Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + # check the embedding status 'before' installing the embedding + embedding_record = library.get_embedding_status() + print("embedding record - before embedding ", embedding_record) + return embedding_record, library + + +@app.cell +def __(mo): + mo.md("\n**Step 2** - Pull down the sample files from S3 through the .load_sample_files() command") + return + + +@app.cell +def __(Setup): + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + return sample_files_path, + + +@app.cell +def __(mo): + mo.md("\n **Step 3** - point .add_files method to the folder of documents that was just created") + return + + +@app.cell +def __(library, os, sample_files_path): + # this method parses the documents, text chunks, and captures in database + + print("update: Parsing and Text Indexing Files") + + library.add_files(input_folder_path=os.path.join(sample_files_path, "Agreements"), + chunk_size=400, max_chunk_size=600, smart_chunking=1) + return + + +@app.cell +def __(mo): + mo.md("\n **2** - Install Vector Embeddings") + return + + +@app.cell +def __(LLMWareConfig, embedding_model, library): + """ This method is the core example of installing an embedding on a library. + -- two inputs - (1) a pre-created library object and (2) the name of an embedding model """ + + library_name_ = library.library_name + vector_db = LLMWareConfig().get_vector_db() + + print(f"\nupdate: Starting the Embedding: " + f"library - {library_name_} - " + f"vector_db - {vector_db} - " + f"model - {embedding_model}") + return library_name_, vector_db + + +@app.cell +def __(Status, embedding_model, library, library_name_, vector_db): + + # *** this is the one key line of code to create the embedding *** + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db,batch_size=100) + + # note: for using llmware as part of a larger application, you can check the real-time status by polling Status() + # --both the EmbeddingHandler and Parsers write to Status() at intervals while processing + update = Status().get_embedding_status(library_name_, embedding_model) + print("update: Embeddings Complete - Status() check at end of embedding - ", update) + + return update, + + +@app.cell +def __(Query, library): + + # Start using the new vector embeddings with Query + sample_query = "incentive compensation" + print("\n\nupdate: Run a sample semantic/vector query: {}".format(sample_query)) + + # queries are constructed by creating a Query object, and passing a library as input + query_results = Query(library).semantic_query(sample_query, result_count=20) + return query_results, sample_query + + +@app.cell +def __(library, query_results): + for i, entries in enumerate(query_results): + + # each query result is a dictionary with many useful keys + + text = entries["text"] + document_source = entries["file_source"] + page_num = entries["page_num"] + vector_distance = entries["distance"] + + # to see all of the dictionary keys returned, uncomment the line below + # print("update: query_results - all - ", i, entries) + + # for display purposes only, we will only show the first 125 characters of the text + if len(text) > 125: text = text[0:125] + " ... " + + print("\nupdate: query results - {} - document - {} - page num - {} - distance - {} " + .format( i, document_source, page_num, vector_distance)) + + print("update: text sample - ", text) + + # lets take a look at the library embedding status again at the end to confirm embeddings were created + embedding_record_ = library.get_embedding_status() + + print("\nupdate: embedding record - ", embedding_record_) + return ( + document_source, + embedding_record_, + entries, + i, + page_num, + text, + vector_distance, + ) + + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example3.py b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example3.py new file mode 100644 index 0000000..4086f39 --- /dev/null +++ b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example3.py @@ -0,0 +1,477 @@ +import marimo + +__generated_with = "0.5.2" +app = marimo.App() + + +@app.cell +def __(): + import marimo as mo + return mo, + + +@app.cell +def __(mo): + mo.md(""" Fast Start Example #3 - Prompts & Model Catalog - how to build prompts and run inferences + + In this example, we will illustrate: + + 1. Discovery - how to discover models in the llmware ModelCatalog + 2. Load Model - how to load a selected model from the catalog + 3. Prompt - how to create a basic prompt and run an inference with the model + + """) + return + + +@app.cell +def __(): + import time + from llmware.prompts import Prompt + from llmware.models import ModelCatalog + return ModelCatalog, Prompt, time + + +@app.cell +def __(mo): + mo.md(""" This is a set of useful test questions to do a 'hello world' but there is nothing special about the + questions - please feel free to edit and ask your own queries with your own context passages. + + --if you are using one of the llmware models, please take note that the models have been trained to answer + based on the information provided, so if you ask a question without passing any context passage, then + don't be surprised if the model responds with 'Not Found.' """) + return + + +@app.cell +def __(): + test_list = [{"query": "What is the total amount of the invoice?", + "answer": "$22,500.00", + "context": "Services Vendor Inc. \n100 Elm Street Pleasantville, NY \nTO Alpha Inc. 5900 1st Street " + "Los Angeles, CA \nDescription Front End Engineering Service $5000.00 \n Back End Engineering" + " Service $7500.00 \n Quality Assurance Manager $10,000.00 \n Total Amount $22,500.00 \n" + "Make all checks payable to Services Vendor Inc. Payment is due within 30 days." + "If you have any questions concerning this invoice, contact Bia Hermes. " + "THANK YOU FOR YOUR BUSINESS! INVOICE INVOICE # 0001 DATE 01/01/2022 FOR Alpha Project P.O. # 1000"},{"query": "What was the amount of the trade surplus?", + "answer": "62.4 billion yen ($416.6 million)", + "context": "Japan’s September trade balance swings into surplus, surprising expectations" + "Japan recorded a trade surplus of 62.4 billion yen ($416.6 million) for September, " + "beating expectations from economists polled by Reuters for a trade deficit of 42.5 " + "billion yen. Data from Japan’s customs agency revealed that exports in September " + "increased 4.3% year on year, while imports slid 16.3% compared to the same period " + "last year. According to FactSet, exports to Asia fell for the ninth straight month, " + "which reflected ongoing China weakness. Exports were supported by shipments to " + "Western markets, FactSet added. — Lim Hui Jie"}, + + {"query": "What was Microsoft's revenue in the 3rd quarter?", + "answer": "$52.9 billion", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "When did the LISP machine market collapse?", + "answer": "1987.", + "context": "The attendees became the leaders of AI research in the 1960s." + " They and their students produced programs that the press described as 'astonishing': " + "computers were learning checkers strategies, solving word problems in algebra, " + "proving logical theorems and speaking English. By the middle of the 1960s, research in " + "the U.S. was heavily funded by the Department of Defense and laboratories had been " + "established around the world. Herbert Simon predicted, 'machines will be capable, " + "within twenty years, of doing any work a man can do'. Marvin Minsky agreed, writing, " + "'within a generation ... the problem of creating 'artificial intelligence' will " + "substantially be solved'. They had, however, underestimated the difficulty of the problem. " + "Both the U.S. and British governments cut off exploratory research in response " + "to the criticism of Sir James Lighthill and ongoing pressure from the US Congress " + "to fund more productive projects. Minsky's and Papert's book Perceptrons was understood " + "as proving that artificial neural networks approach would never be useful for solving " + "real-world tasks, thus discrediting the approach altogether. The 'AI winter', a period " + "when obtaining funding for AI projects was difficult, followed. In the early 1980s, " + "AI research was revived by the commercial success of expert systems, a form of AI " + "program that simulated the knowledge and analytical skills of human experts. By 1985, " + "the market for AI had reached over a billion dollars. At the same time, Japan's fifth " + "generation computer project inspired the U.S. and British governments to restore funding " + "for academic research. However, beginning with the collapse of the Lisp Machine market " + "in 1987, AI once again fell into disrepute, and a second, longer-lasting winter began."}, + + {"query": "When will employment start?", + "answer": "April 16, 2012.", + "context": "THIS EXECUTIVE EMPLOYMENT AGREEMENT (this “Agreement”) is entered " + "into this 2nd day of April, 2012, by and between Aphrodite Apollo " + "(“Executive”) and TestCo Software, Inc. (the “Company” or “Employer”), " + "and shall become effective upon Executive’s commencement of employment " + "(the “Effective Date”) which is expected to commence on April 16, 2012. " + "The Company and Executive agree that unless Executive has commenced " + "employment with the Company as of April 16, 2012 (or such later date as " + "agreed by each of the Company and Executive) this Agreement shall be " + "null and void and of no further effect."}, + + {"query": "What is the current rate on 10-year treasuries?", + "answer": "4.58%", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}, + + {"query": "What is the governing law?", + "answer": "State of Massachusetts", + "context": "19. Governing Law and Procedures. This Agreement shall be governed by and interpreted " + "under the laws of the State of Massachusetts, except with respect to Section 18(a) of this Agreement," + " which shall be governed by the laws of the State of Delaware, without giving effect to any " + "conflict of laws provisions. Employer and Executive each irrevocably and unconditionally " + "(a) agrees that any action commenced by Employer for preliminary and permanent injunctive relief " + "or other equitable relief related to this Agreement or any action commenced by Executive pursuant " + "to any provision hereof, may be brought in the United States District Court for the federal " + "district in which Executive’s principal place of employment is located, or if such court does " + "not have jurisdiction or will not accept jurisdiction, in any court of general jurisdiction " + "in the state and county in which Executive’s principal place of employment is located, " + "(b) consents to the non-exclusive jurisdiction of any such court in any such suit, action o" + "r proceeding, and (c) waives any objection which Employer or Executive may have to the " + "laying of venue of any such suit, action or proceeding in any such court. Employer and " + "Executive each also irrevocably and unconditionally consents to the service of any process, " + "pleadings, notices or other papers in a manner permitted by the notice provisions of Section 8."}, + + {"query": "What is the amount of the base salary?", + "answer": "$200,000.", + "context": "2.2. Base Salary. For all the services rendered by Executive hereunder, during the " + "Employment Period, Employer shall pay Executive a base salary at the annual rate of " + "$200,000, payable semimonthly in accordance with Employer’s normal payroll practices. " + "Executive’s base salary shall be reviewed annually by the Board (or the compensation committee " + "of the Board), pursuant to Employer’s normal compensation and performance review policies " + "for senior level executives, and may be increased but not decreased. The amount of any " + "increase for each year shall be determined accordingly. For purposes of this Agreement, " + "the term “Base Salary” shall mean the amount of Executive’s base salary established " + "from time to time pursuant to this Section 2.2. "}, + + {"query": "Is the expected gross margin greater than 70%?", + "answer": "Yes, between 71.5% and 72.%", + "context": "Outlook NVIDIA’s outlook for the third quarter of fiscal 2024 is as follows:" + "Revenue is expected to be $16.00 billion, plus or minus 2%. GAAP and non-GAAP " + "gross margins are expected to be 71.5% and 72.5%, respectively, plus or minus " + "50 basis points. GAAP and non-GAAP operating expenses are expected to be " + "approximately $2.95 billion and $2.00 billion, respectively. GAAP and non-GAAP " + "other income and expense are expected to be an income of approximately $100 " + "million, excluding gains and losses from non-affiliated investments. GAAP and " + "non-GAAP tax rates are expected to be 14.5%, plus or minus 1%, excluding any discrete items." + "Highlights NVIDIA achieved progress since its previous earnings announcement " + "in these areas: Data Center Second-quarter revenue was a record $10.32 billion, " + "up 141% from the previous quarter and up 171% from a year ago. Announced that the " + "NVIDIA® GH200 Grace™ Hopper™ Superchip for complex AI and HPC workloads is shipping " + "this quarter, with a second-generation version with HBM3e memory expected to ship " + "in Q2 of calendar 2024. "}, + + {"query": "What is Bank of America's rating on Target?", + "answer": "Buy", + "context": "Here are some of the tickers on my radar for Thursday, Oct. 12, taken directly from " + "my reporter’s notebook: It’s the one-year anniversary of the S&P 500′s bear market bottom " + "of 3,577. Since then, as of Wednesday’s close of 4,376, the broad market index " + "soared more than 22%. Hotter than expected September consumer price index, consumer " + "inflation. The Social Security Administration issues announced a 3.2% cost-of-living " + "adjustment for 2024. Chipotle Mexican Grill (CMG) plans price increases. Pricing power. " + "Cites consumer price index showing sticky retail inflation for the fourth time " + "in two years. Bank of America upgrades Target (TGT) to buy from neutral. Cites " + "risk/reward from depressed levels. Traffic could improve. Gross margin upside. " + "Merchandising better. Freight and transportation better. Target to report quarter " + "next month. In retail, the CNBC Investing Club portfolio owns TJX Companies (TJX), " + "the off-price juggernaut behind T.J. Maxx, Marshalls and HomeGoods. Goldman Sachs " + "tactical buy trades on Club names Wells Fargo (WFC), which reports quarter Friday, " + "Humana (HUM) and Nvidia (NVDA). BofA initiates Snowflake (SNOW) with a buy rating." + "If you like this story, sign up for Jim Cramer’s Top 10 Morning Thoughts on the " + "Market email newsletter for free. Barclays cuts price targets on consumer products: " + "UTZ Brands (UTZ) to $16 per share from $17. Kraft Heinz (KHC) to $36 per share from " + "$38. Cyclical drag. J.M. Smucker (SJM) to $129 from $160. Secular headwinds. " + "Coca-Cola (KO) to $59 from $70. Barclays cut PTs on housing-related stocks: Toll Brothers" + "(TOL) to $74 per share from $82. Keeps underweight. Lowers Trex (TREX) and Azek" + "(AZEK), too. Goldman Sachs (GS) announces sale of fintech platform and warns on " + "third quarter of 19-cent per share drag on earnings. The buyer: investors led by " + "private equity firm Sixth Street. Exiting a mistake. Rise in consumer engagement for " + "Spotify (SPOT), says Morgan Stanley. The analysts hike price target to $190 per share " + "from $185. Keeps overweight (buy) rating. JPMorgan loves elf Beauty (ELF). Keeps " + "overweight (buy) rating but lowers price target to $139 per share from $150. " + "Sees “still challenging” environment into third-quarter print. The Club owns shares " + "in high-end beauty company Estee Lauder (EL). Barclays upgrades First Solar (FSLR) " + "to overweight from equal weight (buy from hold) but lowers price target to $224 per " + "share from $230. Risk reward upgrade. Best visibility of utility scale names."}, + + {"query": "Who is NVIDIA's partner for the driver assistance system?", + "answer": "MediaTek", + "context": "Automotive Second-quarter revenue was $253 million, down 15% from the previous " + "quarter and up 15% from a year ago. Announced that NVIDIA DRIVE Orin™ is powering " + "the new XPENG G6 Coupe SUV’s intelligent advanced driver assistance system. " + "Partnered with MediaTek, which will develop mainstream automotive systems on " + "chips for global OEMs, which integrate new NVIDIA GPU chiplet IP for AI and graphics."}, + + {"query": "What was the rate of decline in 3rd quarter sales?", + "answer": "20% year-on-year.", + "context": "Nokia said it would cut up to 14,000 jobs as part of a cost cutting plan following " + "third quarter earnings that plunged. The Finnish telecommunications giant said that " + "it will reduce its cost base and increase operation efficiency to “address the " + "challenging market environment. The substantial layoffs come after Nokia reported " + "third-quarter net sales declined 20% year-on-year to 4.98 billion euros. Profit over " + "the period plunged by 69% year-on-year to 133 million euros."}, + + {"query": "What was professional visualization revenue in the quarter?", + "answer": "$379 million", + "context": "Gaming Second-quarter revenue was $2.49 billion, up 11% from the previous quarter and up " + "22% from a year ago. Began shipping the GeForce RTX™ 4060 family of GPUs, " + "bringing to gamers NVIDIA Ada Lovelace architecture and DLSS, starting at $299." + "Announced NVIDIA Avatar Cloud Engine, or ACE, for Games, a custom AI model " + "foundry service using AI-powered natural language interactions to transform games " + "by bringing intelligence to non-playable characters. Added 35 DLSS games, including " + "Diablo IV, Ratchet & Clank: Rift Apart, Baldur’s Gate 3 and F1 23, as well as Portal: " + "Prelude RTX, a path-traced game made by the community using NVIDIA’s RTX Remix creator tool." + "Professional Visualization Second-quarter revenue was $379 million, up 28% from the " + "previous quarter and down 24% from a year ago. Announced three new desktop " + "workstation RTX GPUs based on the Ada Lovelace architecture — NVIDIA RTX 5000, RTX 4500 " + "and RTX 4000 — to deliver the latest AI, graphics and real-time rendering, which are " + "shipping this quarter. Announced a major release of the NVIDIA Omniverse platform, " + "with new foundation applications and services for developers and industrial " + "enterprises to optimize and enhance their 3D pipelines with OpenUSD and " + "generative AI. Joined with Pixar, Adobe, Apple and Autodesk to form the " + "Alliance for OpenUSD to promote the standardization, development, evolution and " + "growth of Universal Scene Description technology."}, + + + {"query": "What is the executive's title?", + "answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", + "context": "2.1. Duties and Responsibilities and Extent of Service. During the Employment Period, " + "Executive shall serve as Senior Vice President, Event Planning (“SVP”) of the Employer’s " + "Workforce Optimization Division. In such role, Executive will report to the Board of " + "Directors of Employer (the “Board”) and shall devote substantially all of his business time " + "and attention and his best efforts and ability to the operations of Employer and its subsidiaries. " + "Executive shall be responsible for running Employer’s day-to-day operations and shall perform " + "faithfully, diligently and competently the duties and responsibilities of a SVP and such other " + "duties and responsibilities as directed by the Board and are consistent with such position. " + "The foregoing shall not be construed as preventing Executive from (a) making passive " + "investments in other businesses or enterprises consistent with Employer’s code of conduct, " + "or (b) engaging in any other business activity consistent with Employer’s code of conduct; " + "provided that Executive seeks and obtains the prior approval of the Board before engaging " + "in any other business activity. In addition, it shall not be a violation of this Agreement " + "for Executive to participate in civic or charitable activities, deliver lectures, fulfill " + "speaking engagements, teach at educational institutions, and/or manage personal investments " + "(subject to the immediately preceding sentence); provided that such activities do not " + "interfere in any substantial respect with the performance of Executive’s responsibilities " + "as an employee in accordance with this Agreement. Executive may also serve on one or more " + "corporate boards of another company (and committees thereof) upon giving advance notice " + "to the Board prior to commencing service on any other corporate board."}, + + {"query": "According to the CFO, what led to the increase in cloud revenue?", + "answer": "Focused execution by our sales teams and partners", + "context": "'The world's most advanced AI models " + "are coming together with the world's most universal user interface - natural language - " + "to create a new era of computing,' said Satya Nadella, chairman and chief " + "executive officer of Microsoft. 'Across the Microsoft Cloud, we are the platform " + "of choice to help customers get the most value out of their digital spend and innovate " + "for this next generation of AI.' 'Focused execution by our sales teams and partners " + "in this dynamic environment resulted in Microsoft Cloud revenue of $28.5 billion, " + "up 22% (up 25% in constant currency) year-over-year,' said Amy Hood, executive " + "vice president and chief financial officer of Microsoft.\n"}, + + {"query": "Which company is located in Nevada?", + "answer": "North Industries", + "context": "To send notices to Blue Moon Tech, mail to their headquarters at: " + "555 California Street, San Francisco, California 94123. To send notices to North Industries, mail to" + "their principal U.S. offices at: 19832 32nd Avenue, Las Vegas, Nevada 23593.\nTo send notices " + "to Red River Industries, send to: One Red River Road, Stamford, Connecticut 08234."}, + + {"query": "When can termination after a material breach occur?", + "answer": "If the breach is not cured within 15 days of notice of the breach.", + "context": "This Agreement shall remain in effect until terminated. Either party may terminate this " + "agreement, any Statement of Work or Services Description for convenience by giving the other " + "party 30 days written notice. Either party may terminate this Agreement or any work order or " + "services description if the other party is in material breach or default of any obligation " + "that is not cured within 15 days’ notice of such breach. The TestCo agrees to pay all fees " + "for services performed and expenses incurred prior to the termination of this Agreement. " + "Termination of this Agreement will terminate all outstanding Statement of Work or Services " + "Description entered into under this agreement."}, + + {"query": "What is a headline summary in 10 words or less?", + "answer": "Joe Biden is the 46th President of the United States.", + "context": "Joe Biden's tenure as the 46th president of the United States began with " + "his inauguration on January 20, 2021. Biden, a Democrat from Delaware who " + "previously served as vice president under Barack Obama, " + "took office following his victory in the 2020 presidential election over " + "Republican incumbent president Donald Trump. Upon his inauguration, he " + "became the oldest president in American history."}, + + {"query": "Who are the two people that won elections in Georgia?", + "answer": "Jon Ossoff and Raphael Warnock", + "context": "Though Biden was generally acknowledged as the winner, " + "General Services Administration head Emily W. Murphy " + "initially refused to begin the transition to the president-elect, " + "thereby denying funds and office space to his team. " + "On November 23, after Michigan certified its results, Murphy " + "issued the letter of ascertainment, granting the Biden transition " + "team access to federal funds and resources for an orderly transition. " + "Two days after becoming the projected winner of the 2020 election, " + "Biden announced the formation of a task force to advise him on the " + "COVID-19 pandemic during the transition, co-chaired by former " + "Surgeon General Vivek Murthy, former FDA commissioner David A. Kessler, " + "and Yale University's Marcella Nunez-Smith. On January 5, 2021, " + "the Democratic Party won control of the United States Senate, " + "effective January 20, as a result of electoral victories in " + "Georgia by Jon Ossoff in a runoff election for a six-year term " + "and Raphael Warnock in a special runoff election for a two-year term. " + "President-elect Biden had supported and campaigned for both " + "candidates prior to the runoff elections on January 5.On January 6, " + "a mob of thousands of Trump supporters violently stormed the Capitol " + "in the hope of overturning Biden's election, forcing Congress to " + "evacuate during the counting of the Electoral College votes. More " + "than 26,000 National Guard members were deployed to the capital " + "for the inauguration, with thousands remaining into the spring."}, + {"query": "What is the list of the top financial highlights for the quarter?", + "answer": "•Revenue: $52.9 million, up 10% in constant currency;\n" + "•Operating income: $22.4 billion, up 15% in constant currency;\n" + "•Net income: $18.3 billion, up 14% in constant currency;\n" + "•Diluted earnings per share: $2.45 billion, up 14% in constant currency.", + "context": "Microsoft Cloud Strength Drives Third Quarter Results \nREDMOND, Wash. — April 25, 2023 — " + "Microsoft Corp. today announced the following results for the quarter ended March 31, 2023," + " as compared to the corresponding period of last fiscal year:\n· Revenue was $52.9 billion" + " and increased 7% (up 10% in constant currency)\n· Operating income was $22.4 billion " + "and increased 10% (up 15% in constant currency)\n· Net income was $18.3 billion and " + "increased 9% (up 14% in constant currency)\n· Diluted earnings per share was $2.45 " + "and increased 10% (up 14% in constant currency).\n"}, + + {"query": "What is a list of the key points?", + "answer": "•Stocks rallied on Friday with stronger-than-expected U.S jobs data and increase in " + "Treasury yields;\n•Dow Jones gained 195.12 points;\n•S&P 500 added 1.59%;\n•Nasdaq Composite rose " + "1.35%;\n•U.S. economy added 438,000 jobs in August, better than the 273,000 expected;\n" + "•10-year Treasury rate trading near the highest level in 14 years at 4.58%.", + "context": "Stocks rallied Friday even after the release of stronger-than-expected U.S. jobs data " + "and a major increase in Treasury yields. The Dow Jones Industrial Average gained 195.12 points, " + "or 0.76%, to close at 31,419.58. The S&P 500 added 1.59% at 4,008.50. The tech-heavy " + "Nasdaq Composite rose 1.35%, closing at 12,299.68. The U.S. economy added 438,000 jobs in " + "August, the Labor Department said. Economists polled by Dow Jones expected 273,000 " + "jobs. However, wages rose less than expected last month. Stocks posted a stunning " + "turnaround on Friday, after initially falling on the stronger-than-expected jobs report. " + "At its session low, the Dow had fallen as much as 198 points; it surged by more than " + "500 points at the height of the rally. The Nasdaq and the S&P 500 slid by 0.8% during " + "their lowest points in the day. Traders were unclear of the reason for the intraday " + "reversal. Some noted it could be the softer wage number in the jobs report that made " + "investors rethink their earlier bearish stance. Others noted the pullback in yields from " + "the day’s highs. Part of the rally may just be to do a market that had gotten extremely " + "oversold with the S&P 500 at one point this week down more than 9% from its high earlier " + "this year. Yields initially surged after the report, with the 10-year Treasury rate trading " + "near its highest level in 14 years. The benchmark rate later eased from those levels, but " + "was still up around 6 basis points at 4.58%. 'We’re seeing a little bit of a give back " + "in yields from where we were around 4.8%. [With] them pulling back a bit, I think that’s " + "helping the stock market,' said Margaret Jones, chief investment officer at Vibrant Industries " + "Capital Advisors. 'We’ve had a lot of weakness in the market in recent weeks, and potentially " + "some oversold conditions.'"}] + return test_list, + + +@app.cell +def __(mo): + mo.md("""Step 1 - we will pick a model from the ModelCatalog """) + return + + +@app.cell +def __(ModelCatalog): + # A few useful methods to discover and display a list of available models... + + # all generative models + llm_models = ModelCatalog().list_generative_models() + + # if you only want to see the local models + llm_local_models = ModelCatalog().list_generative_local_models() + + # to see only the open source models + llm_open_source_models = ModelCatalog().list_open_source_models() + + # we will print out the local models + for j, models in enumerate(llm_local_models): + print("models: ", j, models["model_name"], models["model_family"]) + + # for purposes of demo, try a few selected models from the list + + model_short_list = ["llmware/bling-1b-0.1", + "llmware/bling-tiny-llama-v0", + "llmware/dragon-yi-6b-gguf", + "llmware/bling-falcon-1b-0.1"] + + model_name = "llmware/bling-1b-0.1" + + # to swap in a gpt-4 openai model - uncomment these two lines + # model_name = "gpt-4" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + return ( + j, + llm_local_models, + llm_models, + llm_open_source_models, + model_name, + model_short_list, + models, + ) + + +@app.cell +def __(mo): + mo.md( """ This is the main example script - it loads the question list, loads the model and executes the prompts. """) + return + + +@app.cell +def __(Prompt, model_name, test_list, time): + t0 = time.time() + + # load in the 'hello world' test questions above + + print(f"\n > Loading Model: {model_name}...") + + prompter = Prompt().load_model(model_name) + + t1 = time.time() + print(f"\n > Model {model_name} load time: {t1-t0} seconds") + + for i, entries in enumerate(test_list): + print(f"\n{i+1}. Query: {entries['query']}") + + # run the prompt + output = prompter.prompt_main(entries["query"], + context=entries["context"], + prompt_name="default_with_context", + temperature=0.30) + + # 'output' is a dictionary with two keys - 'llm_response' and 'usage' + # --'llm_response' is the output from the model + # --'usage' is a dictionary with the usage stats + + llm_response = output["llm_response"].strip("\n") + print(f"LLM Response: {llm_response}") + + # note: the 'gold answer' is the answer we provided above in the hello_world question list + print(f"Gold Answer: {entries['answer']}") + + print(f"LLM Usage: {output['usage']}") + + t2 = time.time() + print(f"\nTotal processing time: {t2-t1} seconds") + + return entries, i, llm_response, output, prompter, t0, t1, t2 + + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example4.py b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example4.py new file mode 100644 index 0000000..ebf9084 --- /dev/null +++ b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example4.py @@ -0,0 +1,256 @@ +import marimo + +__generated_with = "0.5.2" +app = marimo.App() + + +@app.cell +def __(): + import marimo as mo + return mo, + + +@app.cell +def __(mo): + mo.md(""" Fast Start Example #4 - RAG with Text Query + + This example shows a basic RAG recipe using text query combined with LLM prompt. + + We will show two different ways to achieve this basic recipe: + + -- Example 4A - this will integrate Library + Prompt - and is the most scalable general solution + + -- Example 4B - this will illustrate another capability of the Prompt class to add sources "inline" + without necessarily a library in-place. It is another useful tool when you want to be able to quickly + pick up a document and start asking questions to it. + + Note: both of the examples are designed to achieve the same output. + + """) + return + + +@app.cell +def __(): + import os + import re + from llmware.prompts import Prompt, HumanInTheLoop + from llmware.setup import Setup + from llmware.configs import LLMWareConfig + from llmware.retrieval import Query + from llmware.library import Library + return ( + HumanInTheLoop, + LLMWareConfig, + Library, + Prompt, + Query, + Setup, + os, + re, + ) + + +@app.cell +def __(mo): + mo.md("""**Step 0:** Select Model """) + return + + +@app.cell +def __(LLMWareConfig): + # you can pick any model from the ModelCatalog + # we list a few representative good choices below + + LLMWareConfig().set_active_db("sqlite") + + example_models = ["llmware/bling-1b-0.1", "llmware/bling-tiny-llama-v0", "llmware/dragon-yi-6b-gguf"] + + # to swap in a gpt-4 openai model - uncomment these two lines + # model_name = "gpt-4" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + # use local cpu model + model_name = example_models[0] + return example_models, model_name + + +@app.cell +def __(mo): + mo.md("""**Example #4a:** Main general case to run a RAG workflow from a Library """) + return + + +@app.cell +def __(Library, Setup, os): + # Load the llmware sample files + print (f"\n > Loading the llmware sample files...") + sample_files_path_ = Setup().load_sample_files() + contracts_path_ = os.path.join(sample_files_path_,"Agreements") + + contracts_lib = Library().create_new_library("example4_library") + contracts_lib.add_files(contracts_path_) + return contracts_lib, contracts_path_, sample_files_path_ + + +@app.cell +def __(Query, contracts_lib, model_name): + # questions that we want to ask each contract + question_list_ = [{"topic": "executive employment agreement", "llm_query": "What are the names of the two parties?"}, + {"topic": "base salary", "llm_query": "What is the executive's base salary?"}, + {"topic": "governing law", "llm_query": "What is the governing law?"}] + + print (f"\n > Loading model {model_name}...") + + q = Query(contracts_lib) + return q, question_list_ + + +@app.cell +def __(q): + # get a list of all of the unique documents in the library + + # doc id list + doc_list = q.list_doc_id() + print("update: document id list - ", doc_list) + + # filename list + fn_list = q.list_doc_fn() + print("update: filename list - ", fn_list) + return doc_list, fn_list + + +@app.cell +def __(Prompt, doc_list, fn_list, model_name, q, question_list, re): + verbose = False + prompter_ = Prompt().load_model(model_name) + + for i, doc_id in enumerate(doc_list): + + print("\nAnalyzing contract: ", str(i+1), doc_id, fn_list[i]) + + print("LLM Responses:") + + for question in question_list: + + query_topic = question["topic"] + llm_question = question["llm_query"] + + doc_filter = {"doc_ID": [doc_id]} + query_results = q.text_query_with_document_filter(query_topic,doc_filter,result_count=5,exact_mode=True) + + if verbose: + # this will display the query results from the query above + for j, qr in enumerate(query_results): + print("update: querying document - ", query_topic, j, doc_filter, qr) + + source = prompter_.add_source_query_results(query_results) + + # *** this is the call to the llm with the source packaged in the context automatically *** + responses = prompter_.prompt_with_source(llm_question, prompt_name="default_with_context", temperature=0.3) + + # unpacking the results from the LLM + for r, response in enumerate(responses): + print("update: llm response - ", llm_question, re.sub("[\n]"," ", response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter_.clear_source_materials() + return ( + doc_filter, + doc_id, + i, + j, + llm_question, + prompter_, + qr, + query_results, + query_topic, + question, + r, + response, + responses, + source, + verbose, + ) + + +@app.cell +def __(HumanInTheLoop, LLMWareConfig, os, prompter_): + # Save jsonl report to jsonl to /prompt_history folder + print("\nPrompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter_.prompt_id)) + prompter_.save_state() + + # Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review + csv_output = HumanInTheLoop(prompter_).export_current_interaction_to_csv() + print("\nCSV output saved at: ", csv_output) + return csv_output, + + +@app.cell +def __(mo): + mo.md(""" **Example #4b:** Alternative implementation using prompt in-line capabilities without using a library """) + return + + +@app.cell +def __(Prompt, Setup, model_name, os): + # Load the llmware sample files + print(f"\n > Loading the llmware sample files...") + sample_files_path = Setup().load_sample_files() + contracts_path = os.path.join(sample_files_path, "Agreements") + + # questions that we want to ask each contract + question_list = [{"topic": "executive employment agreement", "llm_query": "What are the names of the two parties?"}, + {"topic": "base salary", "llm_query": "What is the executive's base salary?"}, + {"topic": "governing law", "llm_query": "What is the governing law?"}] + + print(f"\n > Loading model {model_name}...") + + prompter = Prompt().load_model(model_name) + return contracts_path, prompter, question_list, sample_files_path + + +@app.cell +def __(contracts_path, os, prompter, question_list, re): + _verbose = False + + for _i, _contract in enumerate(os.listdir(contracts_path)): + + # exclude potential mac os created file artifact in the samples folder path + if _contract != ".DS_Store": + + print("\nAnalyzing contract: ", str(_i + 1), _contract) + + print("LLM Responses:") + + for _question in question_list: + + _query_topic = _question["topic"] + _llm_question = _question["llm_query"] + + # introducing "add_source_document" + # this will perform 'inline' parsing, text chunking and query filter on a document + # input is a file folder path, file name, and an optional query filter + # the source is automatically packaged into the prompt object + + _source = prompter.add_source_document(contracts_path,_contract,query=_query_topic) + + if _verbose: + print("update: document created source - ", _source) + + # calling the LLM with 'source' information from the contract automatically packaged into the prompt + _responses = prompter.prompt_with_source(_llm_question, prompt_name="default_with_context", + temperature=0.3) + + # unpacking the LLM responses + for _r, _response in enumerate(_responses): + print("update: llm response: ", _llm_question, re.sub("[\n]", " ", + _response["llm_response"]).strip()) + + # We're done with this contract, clear the source from the prompt + prompter.clear_source_materials() + return + + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example5.py b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example5.py new file mode 100644 index 0000000..29e20e4 --- /dev/null +++ b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example5.py @@ -0,0 +1,223 @@ +import marimo + +__generated_with = "0.5.2" +app = marimo.App() + + +@app.cell +def __(): + import marimo as mo + return mo, + + +@app.cell +def __(mo): + mo.md(""" Fast Start Example #5 - RAG with Semantic Query + + This example illustrates the most common RAG retrieval pattern, which is using a semantic query, e.g., + a natural language query, as the basis for retrieving relevant text chunks, and then using as + the context material in a prompt to ask the same question to a LLM. + + In this example, we will show the following: + + 1. Create library and install embeddings (feel free to skip / substitute a library created in an earlier step). + 2. Ask a general semantic query to the entire library collection. + 3. Select the most relevant results by document. + 4. Loop through all of the documents - packaging the context and asking our questions to the LLM. + + NOTE: to use chromadb, you may need to install the python sdk: pip3 install chromadb. + + """) + + return + + +@app.cell +def __(): + import os + from llmware.library import Library + from llmware.retrieval import Query + from llmware.setup import Setup + from llmware.status import Status + from llmware.prompts import Prompt + from llmware.configs import LLMWareConfig + from importlib import util + if not util.find_spec("chromadb"): + print("\nto run this example with chromadb, you need to install the chromadb python sdk: pip3 install chromadb") + return LLMWareConfig, Library, Prompt, Query, Setup, Status, os, util + + +@app.cell +def __(LLMWareConfig): + LLMWareConfig().set_active_db("sqlite") + + # for this example, we will use an embedding model that has been 'fine-tuned' for contracts + embedding_model = "industry-bert-contracts" + return embedding_model, + + +@app.cell +def __(): + # note: as of llmware==0.2.12, we have shifted from faiss to chromadb for the Fast Start examples + # --if you are using a Python version before 3.12, please feel free to substitute for "faiss" + # --for versions of Python >= 3.12, for the Fast Start examples (e.g., no install required), we + # recommend using chromadb or lancedb + + # please double-check: `pip3 install chromadb` or pull the latest llmware version to get automatically + # -- if you have installed any other vector db, just change the name, e.g, "milvus" or "pg_vector" + + vector_db = "chromadb" + + # pick any name for the library + library_name = "example_5_library" + + example_models = ["llmware/bling-1b-0.1", "llmware/bling-tiny-llama-v0", "llmware/dragon-yi-6b-gguf"] + + # use local cpu model + llm_model_name = example_models[0] + + # to swap in a gpt-4 openai model - uncomment these two lines + # llm_model_name = "gpt-4" + # os.environ["USER_MANAGED_OPENAI_API_KEY"] = "" + + return example_models, library_name, llm_model_name, vector_db + + +@app.cell +def __(mo): + mo.md(""" **Step 1 -** Create library which is the main 'organizing construct' in llmware """) + return + + +@app.cell +def __(Library, library_name): + print ("\nupdate: Step 1 - Creating library: {}".format(library_name)) + + library = Library().create_new_library(library_name) + + return library, + + +@app.cell +def __(mo): + mo.md(""" **Step 2 -** Pull down the sample files from S3 through the .load_sample_files() command """) + return + + +@app.cell +def __(Setup, os): + # --note: if you need to refresh the sample files, set 'over_write=True' + print ("update: Step 2 - Downloading Sample Files") + + sample_files_path = Setup().load_sample_files(over_write=False) + contracts_path = os.path.join(sample_files_path, "Agreements") + + return contracts_path, sample_files_path + + +@app.cell +def __(mo): + mo.md(""" Step 3 - point ".add_files" method to the folder of documents that was just created """) + return + + +@app.cell +def __(contracts_path, library): + # this method parses all of the documents, text chunks, and captures in MongoDB + print("update: Step 3 - Parsing and Text Indexing Files") + + library.add_files(input_folder_path=contracts_path, chunk_size=400, max_chunk_size=600, + smart_chunking=1) + return + + +@app.cell +def __(mo): + mo.md("""**Step 4** - Install the embeddings """) + return + + +@app.cell +def __(embedding_model, library, vector_db): + print("\nupdate: Step 4 - Generating Embeddings in {} db - with Model- {}".format(vector_db, embedding_model)) + + library.install_new_embedding(embedding_model_name=embedding_model, vector_db=vector_db) + return + + +@app.cell +def __(mo): + mo.md(""" **RAG steps** """) + return + + +@app.cell +def __(Prompt, Query, contracts_path, library, llm_model_name, os): + + print("\nupdate: Loading model for LLM inference - ", llm_model_name) + + prompter = Prompt().load_model(llm_model_name) + + query = "what is the executive's base annual salary" + + # key step: run semantic query against the library and get all of the top results + results = Query(library).semantic_query(query, result_count=50, embedding_distance_threshold=1.0) + + # if you want to look at 'results', uncomment the two lines below + # for i, res in enumerate(results): + # print("update: ", i, res["file_source"], res["distance"], res["text"]) + + for i, contract in enumerate(os.listdir(contracts_path)): + + qr = [] + + if contract != ".DS_Store": + + print("\nContract Name: ", i, contract) + + # we will look through the list of semantic query results, and pull the top results for each file + for j, entries in enumerate(results): + + library_fn = entries["file_source"] + if os.sep in library_fn: + # handles difference in windows file formats vs. mac / linux + library_fn = library_fn.split(os.sep)[-1] + + if library_fn == contract: + print("Top Retrieval: ", j, entries["distance"], entries["text"]) + qr.append(entries) + + # we will add the query results to the prompt + source = prompter.add_source_query_results(query_results=qr) + + # run the prompt + response = prompter.prompt_with_source(query, prompt_name="default_with_context", temperature=0.3) + + # note: prompt_with_resource returns a list of dictionary responses + # -- depending upon the size of the source context, it may call the llm several times + # -- each dict entry represents 1 call to the LLM + + for resp in response: + if "llm_response" in resp: + print("\nupdate: llm answer - ", resp["llm_response"]) + + # start fresh for next document + prompter.clear_source_materials() + return ( + contract, + entries, + i, + j, + library_fn, + prompter, + qr, + query, + resp, + response, + results, + source, + ) + + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example6.py b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example6.py new file mode 100644 index 0000000..d4efacd --- /dev/null +++ b/tutorials/notebooks/fast_start_marimo_examples/llmware_fast_start_example6.py @@ -0,0 +1,173 @@ +import marimo + +__generated_with = "0.5.2" +app = marimo.App() + + +@app.cell +def __(): + import marimo as mo + return mo, + + +@app.cell +def __(mo): + mo.md(""" Fast Start Example #6 - RAG - Beyond the Basics + + This example builds upon examples #4 and #5 and demonstrates how to layer additional elements to + improve the effectiveness of a RAG workflow over a larger set of documents- + + 1. Apply an initial filter across a batch of documents to identify a subset of documents of interest + 2. Analyze the documents of interest to identify key provisions + 3. Use fact-checking and post-processing to validate the accuracy of the LLM response + 4. Write the output to JSON and CSV files for follow-up review and/or the next step in the workflow. + + For this example, we also recommend using a more sophisticated DRAGON model in GGUF format, which enables us + to run 6-7B parameter models locally. + + """) + return + + +@app.cell +def __(): + import os + from llmware.setup import Setup + from llmware.library import Library + from llmware.prompts import Prompt, HumanInTheLoop + from llmware.retrieval import Query + from llmware.configs import LLMWareConfig + + return HumanInTheLoop, LLMWareConfig, Library, Prompt, Query, Setup, os + + +@app.cell +def __(LLMWareConfig): + LLMWareConfig().set_active_db("sqlite") + + # this is part of the DRAGON model series - RAG-fine-tuned fact-based Q&A model + llm_model_name = "llmware/dragon-yi-6b-gguf" + + library_name = "example6_library" + return library_name, llm_model_name + + +@app.cell +def __(mo): + mo.md(""" In this example, we will use the 'AgreementsLarge' sample files which consists of ~80 contracts. We + need to quickly identify the 'master service agreements' as we only want to analyze those contracts. """) + + return + + +@app.cell +def __(Library, Setup, library_name, os): + local_path = Setup().load_sample_files() + agreements_path = os.path.join(local_path, "AgreementsLarge") + + # create a library with all of the Agreements (~80 contracts) + print(f"\nStarting: Parsing 'AgreementsLarge' Folder") + msa_lib = Library().create_new_library(library_name) + msa_lib.add_files(agreements_path) + + return agreements_path, local_path, msa_lib + + +@app.cell +def __(Query, msa_lib): + # find the "master service agreements" (MSA) - we know that 'master services agreement' will always + # be on the first page of the agreement, so we can use that as a good proxy for automatically filtering + # to our target set of documents + + print(f"\nCompleted Parsing - now, let's look for the 'master service agreements', e.g., 'msa'") + + q = Query(msa_lib) + query = '"master services agreement"' + results = q.text_search_by_page(query, page_num=1, results_only=False) + + # results_only = False will return a dictionary with 4 keys: {"query", "results", "doc_ID", "file_source"} + msa_docs = results["file_source"] + msa_doc_ids = results["doc_ID"] + return msa_doc_ids, msa_docs, q, query, results + + +@app.cell +def __(Prompt, llm_model_name, msa_doc_ids): + # load prompt/llm locally + prompter = Prompt().load_model(llm_model_name) + + print("update: identified the following msa doc id: ", msa_doc_ids) + return prompter, + + +@app.cell +def __( + HumanInTheLoop, + LLMWareConfig, + msa_doc_ids, + msa_docs, + os, + prompter, + q, +): + # analyze each MSA - "query" & "llm prompt" + for i, doc_id in enumerate(msa_doc_ids): + + print("\n") + docs = msa_docs[i] + if os.sep in docs: + # handles difference in windows file formats vs. Mac/Linux + docs = docs.split(os.sep)[-1] + + print (i+1, "Reviewing MSA - ", doc_id, docs) + + # look for the termination provisions in each document + doc_filter = {"doc_ID": [doc_id]} + termination_provisions = q.text_query_with_document_filter("termination", doc_filter) + + # package the provisions as a source to a prompt + sources = prompter.add_source_query_results(termination_provisions) + + # if you want to see more details about how the sources are packaged: uncomment this line- + # print("update: sources - ", sources) + + # call the LLM and ask our question + response = prompter.prompt_with_source("What is the notice for termination for convenience?") + + # post processing fact checking + stats = prompter.evidence_comparison_stats(response) + ev_source = prompter.evidence_check_sources(response) + + for i, resp in enumerate(response): + print("update: llm response - ", resp) + print("update: compare with evidence- ", stats[i]["comparison_stats"]) + print("update: sources - ", ev_source[i]["source_review"]) + + prompter.clear_source_materials() + # Save jsonl report with full transaction history to /prompt_history folder + print("\nupdate: Prompt state saved at: ", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id)) + + prompter.save_state() + + # Generate CSV report for easy Human review in Excel + csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv() + + print("\nupdate: CSV output for human review - ", csv_output) + + return ( + csv_output, + doc_filter, + doc_id, + docs, + ev_source, + i, + resp, + response, + sources, + stats, + termination_provisions, + ) + + +if __name__ == "__main__": + app.run() \ No newline at end of file diff --git a/tutorials/notebooks/fix_widgets_jupyter.py b/tutorials/notebooks/fix_widgets_jupyter.py new file mode 100644 index 0000000..abb3aea --- /dev/null +++ b/tutorials/notebooks/fix_widgets_jupyter.py @@ -0,0 +1,23 @@ +import os +import nbformat + +def fix_notebook_metadata(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + nb = nbformat.read(f, as_version=nbformat.NO_CONVERT) + + # Remove broken widgets metadata + if 'widgets' in nb.metadata: + if 'application/vnd.jupyter.widget-state+json' in nb.metadata['widgets']: + widget_meta = nb.metadata['widgets']['application/vnd.jupyter.widget-state+json'] + if 'state' not in widget_meta: + print(f"[FIXING] {file_path}") + widget_meta['state'] = {} + + with open(file_path, 'w', encoding='utf-8') as f: + nbformat.write(nb, f) + +# Walk through all notebooks +for root, _, files in os.walk("../examples/Notebooks"): + for file in files: + if file.endswith(".ipynb"): + fix_notebook_metadata(os.path.join(root, file)) diff --git a/tutorials/notebooks/quickstart_rag_colab.ipynb b/tutorials/notebooks/quickstart_rag_colab.ipynb new file mode 100644 index 0000000..169b834 --- /dev/null +++ b/tutorials/notebooks/quickstart_rag_colab.ipynb @@ -0,0 +1,1426 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "zECN53X4Wf7M" + }, + "source": [ + "**Quick Start for RAG with `llmware`**\n", + "\n", + "This example illustrates a simple contract analysis using a small RAG-optimized LLM running locally\n", + "\n", + "*Note: Colab's built-in Python 3.10 comes with a newer version of `grcpio` (1.60) which is incompatible with a dependency of llmware. Downgrading `grcpio` requires a restart afterwards.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "nGD_UZh4IfYm", + "outputId": "3f74b4f8-1457-4559-faf1-7cf903abbe9e" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Python 3.10.12\n" + ] + } + ], + "source": [ + "!python -V" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "971vCYB51IqI" + }, + "outputs": [], + "source": [ + "%pip install \"grpcio<=1.60.0,>=1.49.1\" --no-cache-dir ## just to be safe with enviroments\n", + "!pip install -q llmware\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "SKBTfwyTDr6F" + }, + "source": [ + "🛑 In Colab click Runtime > Restart session and run all" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RoFBsyUgyJni", + "outputId": "956f1d28-5160-45aa-e53a-00ebe8a60c75" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m22.1/22.1 MB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m796.3/796.3 kB\u001b[0m \u001b[31m29.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.2/128.2 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.5/132.5 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m27.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.6/17.6 MB\u001b[0m \u001b[31m25.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m311.7/311.7 kB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.9/7.9 MB\u001b[0m \u001b[31m34.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m72.0/72.0 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m157.3/157.3 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m671.3/671.3 kB\u001b[0m \u001b[31m30.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.0/86.0 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.3/65.3 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.9/75.9 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.3/9.3 MB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.8/79.8 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m14.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.2/53.2 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m300.4/300.4 kB\u001b[0m \u001b[31m26.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m34.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.8/143.8 kB\u001b[0m \u001b[31m16.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.4/49.4 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Building wheel for ai21 (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Building wheel for word2number (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "llmx 0.0.15a0 requires tiktoken, which is not installed.\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install -q llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "f34b375b7e464941b06fa55056186b1c", + "0878975b308f410a9cd80c1f150f8edc", + "353c350eec234922bcce1fd87b641be9", + "c8238e040b8e4ebfade1b1f84bf3b9d4", + "a1bfe7fb8dcf40f6956aadf34d879711", + "881d89a137ab4a54b629cbaed4a5d116", + "fc4f680743e149e7a329b3028d6aad10", + "a93ea4fd7e314034b832f52766ffa741", + "ad37d5cf8b714c038954998e094d8513", + "b6e5202476c2459a96fd15f7c6e90173", + "49620f0b97394ef6ab921b15beec8069", + "fe76443ff5ae41268a310dea485a30a4", + "ccb677fac5bc48b8ba2c5da953052c93", + "21d7dc2e713c4ccf8b426e2bf16012fe", + "413fbbae136a4c24b7153ef3017325f1", + "5cfb42aa18244f57bcb799c6d17f1347", + "406d86667d234d9aaff0ca844f14e402", + "4f3d19d7e8f24fdb8d39a7a14e302338", + "8f21514be40747f887f161a21698173d", + "c90c25225aac40078311d8ee63a27892", + "a740788a38604222b5594b999b70f0c8", + "f9f989043c0d4312a656bb1f5a15a1da", + "056b709c53a14074a1b295ad66a7e0a0", + "f512d6a27ce245c6918a2f05f1b87a9d", + "8ae4150d5f784bd4a56a4a42fe588b48", + "cc2aa420b4cd44f0988ea1f6ee73d479", + "e0b44f199ee54c58a0bcdb119cec52ef", + "7e8a32daccbf4d7ab23bc5d587faacfd", + "35e2957dc115421bba4a68399a51c9c9", + "9378f057e81148869421d38c6e3d73ce", + "e4c7f46621c945a690c55ed47c34dc77", + "cf0d2867b8304913a2fad9e5115bfde8", + "c113e36bd5704df4b30922533cddc88f" + ] + }, + "id": "cqWRMrztyCfR", + "outputId": "8a22a3a0-09e4-48b0-bd11-44f129abc73d" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " > Loading the llmware sample files...\n", + "\n", + " > Loading model llmware/bling-1b-0.1...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "f34b375b7e464941b06fa55056186b1c", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "config.json: 0%| | 0.00/2.27k [00:00 Loading the llmware sample files...\")\n", + " sample_files_path = Setup().load_sample_files()\n", + " contracts_path = os.path.join(sample_files_path,\"Agreements\")\n", + "\n", + " # query list\n", + " query_list = {\"executive employment agreement\": \"What are the name of the two parties?\",\n", + " \"base salary\": \"What is the executive's base salary?\",\n", + " \"governing law\": \"What is the governing law?\"}\n", + "\n", + " print (f\"\\n > Loading model {model_name}...\")\n", + "\n", + " prompter = Prompt().load_model(model_name)\n", + "\n", + " for i, contract in enumerate(os.listdir(contracts_path)):\n", + "\n", + " # excluding Mac file artifact\n", + " if contract != \".DS_Store\":\n", + "\n", + " print(\"\\nAnalyzing contract: \", str(i+1), contract)\n", + "\n", + " print(\"LLM Responses:\")\n", + " for key, value in query_list.items():\n", + "\n", + " # contract is parsed, text-chunked, and then filtered by topic key\n", + " source = prompter.add_source_document(contracts_path, contract, query=key)\n", + "\n", + " # calling the LLM with 'source' information from the contract automatically packaged into the prompt\n", + " responses = prompter.prompt_with_source(value, prompt_name=\"just_the_facts\", temperature=0.3)\n", + "\n", + " for r, response in enumerate(responses):\n", + " print(key, \":\", re.sub(\"[\\n]\",\" \", response[\"llm_response\"]).strip())\n", + "\n", + " # We're done with this contract, clear the source from the prompt\n", + " prompter.clear_source_materials()\n", + "\n", + " # Save jsonl report to jsonl to /prompt_history folder\n", + " print(\"\\nPrompt state saved at: \", os.path.join(LLMWareConfig.get_prompt_path(),prompter.prompt_id))\n", + " prompter.save_state()\n", + "\n", + " # Save csv report that includes the model, response, prompt, and evidence for human-in-the-loop review\n", + " csv_output = HumanInTheLoop(prompter).export_current_interaction_to_csv()\n", + " print(\"csv output saved at: \", csv_output)\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + "\n", + " # use local cpu model - smallest, fastest (use larger BLING models for higher accuracy)\n", + " model = \"llmware/bling-1b-0.1\"\n", + "\n", + " contract_analysis_on_laptop(model)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "056b709c53a14074a1b295ad66a7e0a0": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f512d6a27ce245c6918a2f05f1b87a9d", + "IPY_MODEL_8ae4150d5f784bd4a56a4a42fe588b48", + "IPY_MODEL_cc2aa420b4cd44f0988ea1f6ee73d479" + ], + "layout": "IPY_MODEL_e0b44f199ee54c58a0bcdb119cec52ef" + } + }, + "0878975b308f410a9cd80c1f150f8edc": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_881d89a137ab4a54b629cbaed4a5d116", + "placeholder": "​", + "style": "IPY_MODEL_fc4f680743e149e7a329b3028d6aad10", + "value": "config.json: 100%" + } + }, + "21d7dc2e713c4ccf8b426e2bf16012fe": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_8f21514be40747f887f161a21698173d", + "max": 4114329821, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_c90c25225aac40078311d8ee63a27892", + "value": 4114329821 + } + }, + "353c350eec234922bcce1fd87b641be9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a93ea4fd7e314034b832f52766ffa741", + "max": 2270, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_ad37d5cf8b714c038954998e094d8513", + "value": 2270 + } + }, + "35e2957dc115421bba4a68399a51c9c9": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "406d86667d234d9aaff0ca844f14e402": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "413fbbae136a4c24b7153ef3017325f1": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a740788a38604222b5594b999b70f0c8", + "placeholder": "​", + "style": "IPY_MODEL_f9f989043c0d4312a656bb1f5a15a1da", + "value": " 4.11G/4.11G [03:24<00:00, 19.9MB/s]" + } + }, + "49620f0b97394ef6ab921b15beec8069": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "4f3d19d7e8f24fdb8d39a7a14e302338": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "5cfb42aa18244f57bcb799c6d17f1347": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "7e8a32daccbf4d7ab23bc5d587faacfd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "881d89a137ab4a54b629cbaed4a5d116": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "8ae4150d5f784bd4a56a4a42fe588b48": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_9378f057e81148869421d38c6e3d73ce", + "max": 2113837, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_e4c7f46621c945a690c55ed47c34dc77", + "value": 2113837 + } + }, + "8f21514be40747f887f161a21698173d": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "9378f057e81148869421d38c6e3d73ce": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a1bfe7fb8dcf40f6956aadf34d879711": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a740788a38604222b5594b999b70f0c8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "a93ea4fd7e314034b832f52766ffa741": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "ad37d5cf8b714c038954998e094d8513": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "b6e5202476c2459a96fd15f7c6e90173": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "c113e36bd5704df4b30922533cddc88f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "c8238e040b8e4ebfade1b1f84bf3b9d4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_b6e5202476c2459a96fd15f7c6e90173", + "placeholder": "​", + "style": "IPY_MODEL_49620f0b97394ef6ab921b15beec8069", + "value": " 2.27k/2.27k [00:00<00:00, 36.9kB/s]" + } + }, + "c90c25225aac40078311d8ee63a27892": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "cc2aa420b4cd44f0988ea1f6ee73d479": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_cf0d2867b8304913a2fad9e5115bfde8", + "placeholder": "​", + "style": "IPY_MODEL_c113e36bd5704df4b30922533cddc88f", + "value": " 2.11M/2.11M [00:00<00:00, 2.13MB/s]" + } + }, + "ccb677fac5bc48b8ba2c5da953052c93": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_406d86667d234d9aaff0ca844f14e402", + "placeholder": "​", + "style": "IPY_MODEL_4f3d19d7e8f24fdb8d39a7a14e302338", + "value": "pytorch_model.bin: 100%" + } + }, + "cf0d2867b8304913a2fad9e5115bfde8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e0b44f199ee54c58a0bcdb119cec52ef": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e4c7f46621c945a690c55ed47c34dc77": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "f34b375b7e464941b06fa55056186b1c": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_0878975b308f410a9cd80c1f150f8edc", + "IPY_MODEL_353c350eec234922bcce1fd87b641be9", + "IPY_MODEL_c8238e040b8e4ebfade1b1f84bf3b9d4" + ], + "layout": "IPY_MODEL_a1bfe7fb8dcf40f6956aadf34d879711" + } + }, + "f512d6a27ce245c6918a2f05f1b87a9d": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7e8a32daccbf4d7ab23bc5d587faacfd", + "placeholder": "​", + "style": "IPY_MODEL_35e2957dc115421bba4a68399a51c9c9", + "value": "tokenizer.json: 100%" + } + }, + "f9f989043c0d4312a656bb1f5a15a1da": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fc4f680743e149e7a329b3028d6aad10": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } + }, + "fe76443ff5ae41268a310dea485a30a4": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_ccb677fac5bc48b8ba2c5da953052c93", + "IPY_MODEL_21d7dc2e713c4ccf8b426e2bf16012fe", + "IPY_MODEL_413fbbae136a4c24b7153ef3017325f1" + ], + "layout": "IPY_MODEL_5cfb42aa18244f57bcb799c6d17f1347" + } + }, + "state": {} + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/tutorials/notebooks/web_services_slim_fx_nb.ipynb b/tutorials/notebooks/web_services_slim_fx_nb.ipynb new file mode 100644 index 0000000..d9e6927 --- /dev/null +++ b/tutorials/notebooks/web_services_slim_fx_nb.ipynb @@ -0,0 +1,2307 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "5DKDTnMGHTGn" + }, + "source": [ + "# If you are using Colab for free, we highly recommend you activate the T4 GPU\n", + "# hardware accelerator. Our models are designed to run with at least 16GB\n", + "# of RAM, activating T4 will grant the notebook 16GB of GDDR6 RAM as opposed\n", + "# to the ~13GB Colab gives automatically.\n", + "# To activate T4:\n", + "# 1. click on the \"Runtime\" tab\n", + "# 2. click on \"Change runtime type\"\n", + "# 3. select T4 GPU under Hardware Accelerator\n", + "# NOTE: there is a weekly usage limit on using T4 for free" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iZ0qxGqFFHrz" + }, + "source": [ + "\n", + "\"\"\" This example illustrates a more complex recipe to generate a structured financial research dictionary with\n", + " 30 keys and values produced, using a combination of models and web services:\n", + "\n", + " Models\n", + " 1. slim-extract-tool\n", + " 2. slim-summary-tool\n", + " 3. bling-stablelm-3b-tool\n", + "\n", + " Web Services\n", + " 1. Yfinance - stock ticker\n", + " 2. Wikipedia - company background information\n", + "\n", + " The example shows how to extract keys from one source that can then be used as a lookup in a web service to\n", + " supplement the original source materials, and provide a secondary source, which can then also be prompted and\n", + " used to extract, analyze and summarize key information.\n", + "\n", + " \"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "cq_s3hGOR4lh", + "outputId": "2d7cb90c-3cc7-4984-c58c-b1810421ec79" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: YFinance in /usr/local/lib/python3.10/dist-packages (0.2.40)\n", + "Requirement already satisfied: pandas>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from YFinance) (2.0.3)\n", + "Requirement already satisfied: numpy>=1.16.5 in /usr/local/lib/python3.10/dist-packages (from YFinance) (1.25.2)\n", + "Requirement already satisfied: requests>=2.31 in /usr/local/lib/python3.10/dist-packages (from YFinance) (2.31.0)\n", + "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.10/dist-packages (from YFinance) (0.0.11)\n", + "Requirement already satisfied: lxml>=4.9.1 in /usr/local/lib/python3.10/dist-packages (from YFinance) (4.9.4)\n", + "Requirement already satisfied: platformdirs>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from YFinance) (4.2.2)\n", + "Requirement already satisfied: pytz>=2022.5 in /usr/local/lib/python3.10/dist-packages (from YFinance) (2023.4)\n", + "Requirement already satisfied: frozendict>=2.3.4 in /usr/local/lib/python3.10/dist-packages (from YFinance) (2.4.4)\n", + "Requirement already satisfied: peewee>=3.16.2 in /usr/local/lib/python3.10/dist-packages (from YFinance) (3.17.5)\n", + "Requirement already satisfied: beautifulsoup4>=4.11.1 in /usr/local/lib/python3.10/dist-packages (from YFinance) (4.12.3)\n", + "Requirement already satisfied: html5lib>=1.1 in /usr/local/lib/python3.10/dist-packages (from YFinance) (1.1)\n", + "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4>=4.11.1->YFinance) (2.5)\n", + "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.10/dist-packages (from html5lib>=1.1->YFinance) (1.16.0)\n", + "Requirement already satisfied: webencodings in /usr/local/lib/python3.10/dist-packages (from html5lib>=1.1->YFinance) (0.5.1)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->YFinance) (2.8.2)\n", + "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas>=1.3.0->YFinance) (2024.1)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->YFinance) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->YFinance) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->YFinance) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.31->YFinance) (2024.6.2)\n", + "Collecting wikipedia-api\n", + " Downloading Wikipedia_API-0.6.0-py3-none-any.whl (14 kB)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from wikipedia-api) (2.31.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (3.7)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->wikipedia-api) (2024.6.2)\n", + "Installing collected packages: wikipedia-api\n", + "Successfully installed wikipedia-api-0.6.0\n", + "Collecting llmware\n", + " Downloading llmware-0.3.2-py3-none-any.whl (56.0 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.0/56.0 MB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting boto3>=1.24.53 (from llmware)\n", + " Downloading boto3-1.34.138-py3-none-any.whl (139 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.2/139.2 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.23.4)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.10/dist-packages (from llmware) (1.25.2)\n", + "Collecting pymongo>=4.7.0 (from llmware)\n", + " Downloading pymongo-4.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.2 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m32.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: tokenizers>=0.15.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.19.1)\n", + "Collecting psycopg-binary==3.1.17 (from llmware)\n", + " Downloading psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.3 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.3/3.3 MB\u001b[0m \u001b[31m46.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting psycopg==3.1.17 (from llmware)\n", + " Downloading psycopg-3.1.17-py3-none-any.whl (178 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m178.0/178.0 kB\u001b[0m \u001b[31m20.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pgvector==0.2.4 (from llmware)\n", + " Downloading pgvector-0.2.4-py2.py3-none-any.whl (9.6 kB)\n", + "Collecting colorama==0.4.6 (from llmware)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: librosa>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from llmware) (0.10.2.post1)\n", + "Requirement already satisfied: typing-extensions>=4.1 in /usr/local/lib/python3.10/dist-packages (from psycopg==3.1.17->llmware) (4.12.2)\n", + "Collecting botocore<1.35.0,>=1.34.138 (from boto3>=1.24.53->llmware)\n", + " Downloading botocore-1.34.138-py3-none-any.whl (12.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.4/12.4 MB\u001b[0m \u001b[31m94.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jmespath<2.0.0,>=0.7.1 (from boto3>=1.24.53->llmware)\n", + " Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n", + "Collecting s3transfer<0.11.0,>=0.10.0 (from boto3>=1.24.53->llmware)\n", + " Downloading s3transfer-0.10.2-py3-none-any.whl (82 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.7/82.7 kB\u001b[0m \u001b[31m11.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (3.15.4)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2023.6.0)\n", + "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (24.1)\n", + "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (6.0.1)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->llmware) (4.66.4)\n", + "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (3.0.1)\n", + "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.11.4)\n", + "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.2.2)\n", + "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.4.2)\n", + "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (4.4.2)\n", + "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.58.1)\n", + "Requirement already satisfied: soundfile>=0.12.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.12.1)\n", + "Requirement already satisfied: pooch>=1.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.8.2)\n", + "Requirement already satisfied: soxr>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.3.7)\n", + "Requirement already satisfied: lazy-loader>=0.1 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (0.4)\n", + "Requirement already satisfied: msgpack>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa>=0.10.0->llmware) (1.0.8)\n", + "Collecting dnspython<3.0.0,>=1.16.0 (from pymongo>=4.7.0->llmware)\n", + " Downloading dnspython-2.6.1-py3-none-any.whl (307 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.7/307.7 kB\u001b[0m \u001b[31m39.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.138->boto3>=1.24.53->llmware) (2.8.2)\n", + "Requirement already satisfied: urllib3!=2.2.0,<3,>=1.25.4 in /usr/local/lib/python3.10/dist-packages (from botocore<1.35.0,>=1.34.138->boto3>=1.24.53->llmware) (2.0.7)\n", + "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa>=0.10.0->llmware) (0.41.1)\n", + "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.1->librosa>=0.10.0->llmware) (4.2.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (3.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.19.4->llmware) (2024.6.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa>=0.10.0->llmware) (3.5.0)\n", + "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile>=0.12.1->librosa>=0.10.0->llmware) (1.16.0)\n", + "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile>=0.12.1->librosa>=0.10.0->llmware) (2.22)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.35.0,>=1.34.138->boto3>=1.24.53->llmware) (1.16.0)\n", + "Installing collected packages: psycopg-binary, psycopg, pgvector, jmespath, dnspython, colorama, pymongo, botocore, s3transfer, boto3, llmware\n", + "Successfully installed boto3-1.34.138 botocore-1.34.138 colorama-0.4.6 dnspython-2.6.1 jmespath-1.0.1 llmware-0.3.2 pgvector-0.2.4 psycopg-3.1.17 psycopg-binary-3.1.17 pymongo-4.8.0 s3transfer-0.10.2\n" + ] + } + ], + "source": [ + "!pip install YFinance\n", + "!pip install wikipedia-api\n", + "!pip install llmware" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000, + "referenced_widgets": [ + "2a5570ef8bbd4d9cbc965e1a1872daa0", + "02d42d1c0d104a849c09c14a14ddd774", + "42fb8aaca8114edaba0853c28741e199", + "332c64b2fead4a2c8d8f65690d74a22e", + "a750d844c86148bfb5b39c34b26b42a3", + "6262dbdd76f5406d977c035a9747f6f8", + "cd294ad0b5c14af59be1106700179200", + "02ff0f346ade4f7190d158598916bdf1", + "e3533899a9d34bed9669d72c74f28933", + "bac1b3f941c84acba980a62395981e00", + "e413397fb74942158c9fe27b3a2dc3cc", + "fc5c544522b4494b8526088d092fdf31", + "d6a1ad8c1eb44f9a9e95c90f67bc8137", + "d72368cf2dfe4b67bb73f238507d05c6", + "b878681634cb4505b33a5dc139abb06d", + "0a2d8a7481604e5ea53b3353ed96a8b4", + "19afff5f83c04aea95de1828c2c50690", + "14fd6f274e3d4c609c46de517184c0d1", + "72a003a867dd44e2aad1521b8bf20da3", + "c90fb61f65954b209fcd8d6043031311", + "dbe6788d396a42228aa2699dced0e481", + "a50bfb38fbf64413a7a0125a3d157dd8", + "403a012747b44a6fbb5b570bf98d5a71", + "425e9c03414d4d62864dbe20af462e89", + "227a70e7abdf4a34b796e110848b089e", + "e189c705be104b6e8f480d62ab16fde0", + "0c975ae83ad94f259b5b9e0b76fbb2a9", + "9d06a33c549a4199b4af3f1f48ff5fd1", + "8004072db97e460f83503c899fac9a01", + "713b2d8c0681493c95196f4d56851ef7", + "02b9d8c1cd154c828d7307b4b056b703", + "ee34da33d0ae412f812f132358ab22f1", + "97d5361038274f15aaec50e9736f96ef", + "92b2eb8c11a548b7b4d0e09f8a6e6802", + "2b134ea124b64b838f7ba7b9f461a26b", + "1f2909f9240345ed9ed892f24c57a42f", + "bb6b60b1ac6841708c0bd7497dbd2976", + "b8d4e49b68dd48a49072794539d822fc", + "0e518c2dfd6247c0b4db11b93404630b", + "0ddc467890d74624a5c6c731121dc52c", + "57778bdaee4b47038ed72c41968e25e9", + "4517b4a70120428c934dd9f1647a107d", + "8410ee6ef7c34ef7a8d2c94c78d8f95a", + "f4a0f46c7fb344c881b470fb96150d12", + "ec7d3edf94a744518d4a71df9ec7f5a3", + "853f66999986457cb1d52d8622f02cb8", + "4e0a1748248843d8978f6e224a09894f", + "7f41c49ce2ff417e8db0e178ae70e457", + "bd79a5726d544a06aa891f08d12de28b", + "ee4554c9acf749e2b6ccb81685277cac", + "89a269f8b10e4db6a7d22cfb59797b53", + "0d155d9516104c72bb8047e53aa3ae2f", + "ef74d1ac40d44268b6930a456e1a1fc4", + "21e242b1386147e8a2f7b78113e5bd5b", + "80c197dbbc114735b9d62f0236d534da" + ] + }, + "id": "ZxHLecaSSrYl", + "outputId": "9ee291df-a17b-4121-878b-4a9a62b7e1a3" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:llmware.models:ModelCatalog - load_model - fetching model - bling-stablelm-3b-tool - from remote repository using pull_snapshot_from_hf - this may take a couple of minutes the first time.\n", + "/usr/local/lib/python3.10/dist-packages/huggingface_hub/file_download.py:1194: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\n", + "For more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n", + " warnings.warn(\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2a5570ef8bbd4d9cbc965e1a1872daa0", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Fetching 4 files: 0%| | 0/4 [00:00