{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "\"Open\n", "\n", "# Multi-Modal LLM using NVIDIA endpoints for image reasoning\n", "\n", "In this notebook, we show how to use NVIDIA MultiModal LLM class/abstraction for image understanding/reasoning.\n", "\n", "We also show several functions we are now supporting for NVIDIA LLM:\n", "* `complete` (both sync and async): for a single prompt and list of images\n", "* `stream complete` (both sync and async): for steaming output of complete" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%pip install --upgrade --quiet llama-index-multi-modal-llms-nvidia llama-index-embeddings-nvidia llama-index-readers-file" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import getpass\n", "import os\n", "\n", "# del os.environ['NVIDIA_API_KEY'] ## delete key and reset\n", "if os.environ.get(\"NVIDIA_API_KEY\", \"\").startswith(\"nvapi-\"):\n", " print(\"Valid NVIDIA_API_KEY already in environment. Delete to reset\")\n", "else:\n", " nvapi_key = getpass.getpass(\"NVAPI Key (starts with nvapi-): \")\n", " assert nvapi_key.startswith(\n", " \"nvapi-\"\n", " ), f\"{nvapi_key[:5]}... is not a valid key\"\n", " os.environ[\"NVIDIA_API_KEY\"] = nvapi_key" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import nest_asyncio\n", "\n", "nest_asyncio.apply()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from llama_index.multi_modal_llms.nvidia import NVIDIAMultiModal\n", "import base64\n", "from llama_index.core.schema import ImageDocument\n", "from PIL import Image\n", "import requests\n", "from io import BytesIO\n", "\n", "# import matplotlib.pyplot as plt\n", "from llama_index.core.multi_modal_llms.generic_utils import load_image_urls\n", "\n", "llm = NVIDIAMultiModal()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Initialize `NVIDIAMultiModal` and Load Images from URLs" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "image_urls = [\n", " \"https://res.cloudinary.com/hello-tickets/image/upload/c_limit,f_auto,q_auto,w_1920/v1640835927/o3pfl41q7m5bj8jardk0.jpg\",\n", " \"https://www.visualcapitalist.com/wp-content/uploads/2023/10/US_Mortgage_Rate_Surge-Sept-11-1.jpg\",\n", " \"https://www.sportsnet.ca/wp-content/uploads/2023/11/CP1688996471-1040x572.jpg\",\n", " # Add yours here!\n", "]\n", "\n", "img_response = requests.get(image_urls[0])\n", "img = Image.open(BytesIO(img_response.content))\n", "# plt.imshow(img)\n", "\n", "image_url_documents = load_image_urls(image_urls)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Complete a prompt with a bunch of images" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "response = llm.complete(\n", " prompt=f\"What is this image?\",\n", " image_documents=image_url_documents,\n", ")\n", "\n", "print(response)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "await llm.acomplete(\n", " prompt=\"tell me about this image\",\n", " image_documents=image_url_documents,\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Steam Complete a prompt with a bunch of images" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stream_complete_response = llm.stream_complete(\n", " prompt=f\"What is this image?\",\n", " image_documents=image_url_documents,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for r in stream_complete_response:\n", " print(r.text, end=\"\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "stream_complete_response = await llm.astream_complete(\n", " prompt=f\"What is this image?\",\n", " image_documents=image_url_documents,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "last_element = None\n", "async for last_element in stream_complete_response:\n", " pass\n", "\n", "print(last_element)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Passing an image as a base64 encoded string" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "imgr_content = base64.b64encode(\n", " requests.get(\n", " \"https://helloartsy.com/wp-content/uploads/kids/cats/how-to-draw-a-small-cat/how-to-draw-a-small-cat-step-6.jpg\"\n", " ).content\n", ").decode(\"utf-8\")\n", "\n", "llm.complete(\n", " prompt=\"List models in image\",\n", " image_documents=[ImageDocument(image=imgr_content, mimetype=\"jpeg\")],\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Passing an image as an NVCF asset\n", "If your image is sufficiently large or you will pass it multiple times in a chat conversation, you may upload it once and reference it in your chat conversation\n", "\n", "See https://docs.nvidia.com/cloud-functions/user-guide/latest/cloud-function/assets.html for details about how upload the image." ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import requests\n", "\n", "content_type = \"image/jpg\"\n", "description = \"example-image-from-lc-nv-ai-e-notebook\"\n", "\n", "create_response = requests.post(\n", " \"https://api.nvcf.nvidia.com/v2/nvcf/assets\",\n", " headers={\n", " \"Authorization\": f\"Bearer {os.environ['NVIDIA_API_KEY']}\",\n", " \"accept\": \"application/json\",\n", " \"Content-Type\": \"application/json\",\n", " },\n", " json={\"contentType\": content_type, \"description\": description},\n", ")\n", "create_response.raise_for_status()\n", "\n", "upload_response = requests.put(\n", " create_response.json()[\"uploadUrl\"],\n", " headers={\n", " \"Content-Type\": content_type,\n", " \"x-amz-meta-nvcf-asset-description\": description,\n", " },\n", " data=img_response.content,\n", ")\n", "upload_response.raise_for_status()\n", "\n", "asset_id = create_response.json()[\"assetId\"]\n", "asset_id" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "response = llm.stream_complete(\n", " prompt=f\"Describe the image\",\n", " image_documents=[\n", " ImageDocument(metadata={\"asset_id\": asset_id}, mimetype=\"png\")\n", " ],\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for r in response:\n", " print(r.text, end=\"\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Passing images from local files" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from llama_index.core import SimpleDirectoryReader\n", "\n", "# put your local directore here\n", "image_documents = SimpleDirectoryReader(\"./tests/data/\").load_data()\n", "\n", "llm.complete(\n", " prompt=\"Describe the images as an alternative text\",\n", " image_documents=image_documents,\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Chat with of images" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from llama_index.core.llms import ChatMessage\n", "\n", "llm.chat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=[\n", " {\"type\": \"text\", \"text\": \"Describe this image:\"},\n", " {\"type\": \"image_url\", \"image_url\": image_urls[1]},\n", " ],\n", " )\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from llama_index.core.llms import ChatMessage\n", "\n", "await llm.achat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=[\n", " {\"type\": \"text\", \"text\": \"Describe this image:\"},\n", " {\"type\": \"image_url\", \"image_url\": image_urls[1]},\n", " ],\n", " )\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "llm.chat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=[\n", " {\"type\": \"text\", \"text\": \"Describe the image\"},\n", " {\n", " \"type\": \"image_url\",\n", " \"image_url\": f'',\n", " },\n", " ],\n", " )\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "await llm.achat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=[\n", " {\"type\": \"text\", \"text\": \"Describe the image\"},\n", " {\n", " \"type\": \"image_url\",\n", " \"image_url\": f'',\n", " },\n", " ],\n", " )\n", " ]\n", ")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Stream Chat a prompt with images" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from llama_index.core.llms import ChatMessage\n", "\n", "streaming_resp = llm.stream_chat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=[\n", " {\"type\": \"text\", \"text\": \"Describe this image:\"},\n", " {\"type\": \"image_url\", \"image_url\": image_urls[1]},\n", " ],\n", " )\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for r in streaming_resp:\n", " print(r.delta, end=\"\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from llama_index.core.llms import ChatMessage\n", "\n", "resp = await llm.astream_chat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=[\n", " {\"type\": \"text\", \"text\": \"Describe this image:\"},\n", " {\"type\": \"image_url\", \"image_url\": image_urls[0]},\n", " ],\n", " )\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "last_element = None\n", "async for last_element in resp:\n", " pass\n", "\n", "print(last_element)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "response = llm.stream_chat(\n", " [\n", " ChatMessage(\n", " role=\"user\",\n", " content=f\"\"\"\"\"\",\n", " )\n", " ]\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "for r in response:\n", " print(r.delta, end=\"\")" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3" } }, "nbformat": 4, "nbformat_minor": 2 }