chore: import upstream snapshot with attribution

examples/rest/python/sample_client.py

@@ -0,0 +1,51 @@
+import json
+
+import requests
+
+
+class color:
+    PURPLE = "\033[95m"
+    CYAN = "\033[96m"
+    DARKCYAN = "\033[36m"
+    BLUE = "\033[94m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    END = "\033[0m"
+
+
+# Get a response using a prompt without streaming
+payload = {
+    "model": "vicuna-v1-7b",
+    "messages": [{"role": "user", "content": "Write a haiku"}],
+    "stream": False,
+}
+r = requests.post("http://127.0.0.1:8000/v1/chat/completions", json=payload)
+print(
+    f"{color.BOLD}Without streaming:{color.END}\n{color.GREEN}{r.json()['choices'][0]['message']['content']}{color.END}\n"  # noqa: E501
+)
+
+# Reset the chat
+r = requests.post("http://127.0.0.1:8000/chat/reset", json=payload)
+print(f"{color.BOLD}Reset chat:{color.END} {str(r)}\n")
+
+# Get a response using a prompt with streaming
+payload = {
+    "model": "vicuna-v1-7b",
+    "messages": [{"role": "user", "content": "Write a haiku"}],
+    "stream": True,
+}
+with requests.post("http://127.0.0.1:8000/v1/chat/completions", json=payload, stream=True) as r:
+    print(f"{color.BOLD}With streaming:{color.END}")
+    for chunk in r:
+        if chunk[6:].decode("utf-8").strip() == "[DONE]":
+            break
+        content = json.loads(chunk[6:])["choices"][0]["delta"].get("content", "")
+        print(f"{color.GREEN}{content}{color.END}", end="", flush=True)
+    print("\n")
+
+# Get the latest runtime stats
+r = requests.get("http://127.0.0.1:8000/stats")
+print(f"{color.BOLD}Runtime stats:{color.END} {r.json()}\n")

examples/rest/python/sample_langchain.py

@@ -0,0 +1,168 @@
+from langchain import LLMChain, PromptTemplate
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.chains import RetrievalQA
+from langchain.chains.question_answering import load_qa_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.document_loaders import (
+    DirectoryLoader,
+    TextLoader,
+    UnstructuredRSTLoader,
+)
+from langchain.llms import OpenAI
+from langchain.memory import ConversationBufferWindowMemory
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.vectorstores import Chroma
+
+# Note that Langchain support for embedding documents using MLC is currently blocked on
+# https://github.com/langchain-ai/langchain/pull/7815
+# We have subclassed `OpenAIEmbeddings` in the meantime to get around this dependency.
+from mlc_llm.contrib.embeddings.openai import MLCEmbeddings
+
+# First set the following in your environment:
+# export OPENAI_API_BASE=http://127.0.0.1:8000/v1
+# export OPENAI_API_KEY=EMPTY
+
+# Note that Langchain does not currently support Pydantic v2:
+# https://github.com/langchain-ai/langchain/issues/6841
+# Please ensure that your `pydantic` version is < 2.0
+
+
+class color:
+    PURPLE = "\033[95m"
+    CYAN = "\033[96m"
+    DARKCYAN = "\033[36m"
+    BLUE = "\033[94m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    END = "\033[0m"
+
+
+def llm_chain_example():
+    template = """
+    {history}
+    USER: {human_input}
+    ASSISTANT:"""
+
+    prompt = PromptTemplate(input_variables=["history", "human_input"], template=template)
+
+    llm_chain = LLMChain(
+        llm=ChatOpenAI(streaming=True, callbacks=[StreamingStdOutCallbackHandler()]),
+        prompt=prompt,
+        verbose=True,
+        memory=ConversationBufferWindowMemory(human_prefix="USER", ai_prefix="ASSISTANT"),
+    )
+
+    llm_chain.predict(human_input="Write a short poem about Pittsburgh.")
+    llm_chain.predict(human_input="What does the poem mean?")
+
+
+def load_qa_chain_example():
+    loader = TextLoader("../resources/linux.txt")
+    documents = loader.load()
+    chain = load_qa_chain(llm=OpenAI(), chain_type="stuff", verbose=False)
+    query = "When was Linux released?"
+    print(f"{color.BOLD}Query:{color.END} {color.BLUE} {query}{color.END}")
+    print(
+        f"{color.BOLD}Response:{color.END} {color.GREEN}{chain.run(input_documents=documents, question=query)}{color.END}"  # noqa: E501
+    )
+
+
+def retrieval_qa_sotu_example():
+    prompt_template = """Use only the following pieces of context to answer the question at the end. Don't use any other knowledge.
+
+    {context}
+
+    USER: {question}
+    ASSISTANT:"""  # noqa: E501
+
+    PROMPT = PromptTemplate(template=prompt_template, input_variables=["context", "question"])
+
+    loader = TextLoader("../resources/state_of_the_union.txt")
+    documents = loader.load()
+
+    text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=100)
+    texts = text_splitter.split_documents(documents)
+    # print(texts)
+    embeddings = MLCEmbeddings(deployment="text-embedding-ada-002", embedding_ctx_length=None)
+    db = Chroma.from_documents(documents=texts, embedding=embeddings)
+    retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": 2})
+    qa = RetrievalQA.from_chain_type(
+        llm=OpenAI(),
+        chain_type="stuff",
+        retriever=retriever,
+        return_source_documents=True,
+        chain_type_kwargs={"prompt": PROMPT},
+    )
+    questions = [
+        "What is the American Rescue Plan?",
+        "What did the president say about Ketanji Brown Jackson?",
+        "Who is mentioned in the speech?",
+        "To whom is the speech addressed?",
+        "Tell me more about the Made in America campaign.",
+    ]
+
+    for qn in questions:
+        print(f"{color.BOLD}QUESTION:{color.END} {qn}")
+        res = qa({"query": qn})
+        print(f"{color.BOLD}RESPONSE:{color.END} {color.GREEN}{res['result']}{color.END}")
+        print(
+            f"{color.BOLD}SOURCE:{color.END} {color.BLUE}{repr(res['source_documents'][0].page_content)}{color.END}"  # noqa: E501
+        )
+        print()
+
+
+def retrieval_qa_mlc_docs_example():
+    prompt_template = """Use only the following pieces of context to answer the question at the end. Don't use any other knowledge.
+
+    {context}
+
+    USER: {question}
+    ASSISTANT:"""  # noqa: E501
+
+    PROMPT = PromptTemplate(template=prompt_template, input_variables=["context", "question"])
+
+    loader = DirectoryLoader(
+        "../../../docs",
+        glob="*/*.rst",
+        show_progress=True,
+        loader_cls=UnstructuredRSTLoader,
+        loader_kwargs={"mode": "single"},
+    )
+    documents = loader.load()
+    text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+    texts = text_splitter.split_documents(documents)
+    embeddings = MLCEmbeddings(deployment="text-embedding-ada-002", embedding_ctx_length=None)
+    db = Chroma.from_documents(collection_name="abc", documents=texts, embedding=embeddings)
+    retriever = db.as_retriever(search_type="similarity", search_kwargs={"k": 3})
+    qa = RetrievalQA.from_chain_type(
+        llm=OpenAI(),
+        chain_type="stuff",
+        retriever=retriever,
+        return_source_documents=True,
+        chain_type_kwargs={"prompt": PROMPT},
+    )
+    while True:
+        qn = input(f"{color.BOLD}QUESTION:{color.END} ")
+        res = qa({"query": qn})
+        print(f"{color.BOLD}RESPONSE:{color.END} {color.GREEN}{res['result']}{color.END}")
+        print(
+            f"{color.BOLD}SOURCE:{color.END} {color.BLUE}{repr(res['source_documents'][0].page_content)}{color.END}"  # noqa: E501
+        )
+        print()
+
+    # Some example questions:
+    # - What is the chat config?
+    # - What is temperature?
+    # - What are the REST API endpoints?
+    # - What are the available quantization options?
+
+
+# Uncomment one of the following lines to try out the corresponding demo:
+
+# llm_chain_example()
+# load_qa_chain_example()
+# retrieval_qa_sotu_example()
+# retrieval_qa_mlc_docs_example()

examples/rest/python/sample_openai.py

@@ -0,0 +1,42 @@
+import openai
+
+openai.api_key = "None"
+openai.api_base = "http://127.0.0.1:8000/v1"
+
+model = "vicuna-v1-7b"
+
+
+class color:
+    PURPLE = "\033[95m"
+    CYAN = "\033[96m"
+    DARKCYAN = "\033[36m"
+    BLUE = "\033[94m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    END = "\033[0m"
+
+
+# Chat completion example without streaming
+print(f"{color.BOLD}OpenAI chat completion example without streaming:{color.END}\n")
+completion = openai.ChatCompletion.create(
+    model=model, messages=[{"role": "user", "content": "Write a poem about OpenAI"}]
+)
+print(f"{color.GREEN}{completion.choices[0].message.content}{color.END}\n\n")
+
+# Chat completion example with streaming
+print(f"{color.BOLD}OpenAI chat completion example with streaming:{color.END}\n")
+res = openai.ChatCompletion.create(
+    model=model, messages=[{"role": "user", "content": "Write a poem about OpenAI"}], stream=True
+)
+for chunk in res:
+    content = chunk["choices"][0]["delta"].get("content", "")
+    print(f"{color.GREEN}{content}{color.END}", end="", flush=True)
+print("\n")
+
+# Completion example
+print(f"{color.BOLD}OpenAI completion example:{color.END}\n")
+res = openai.Completion.create(prompt="Write a poem about OpenAI", model=model)
+print(f"{color.GREEN}{res.choices[0].text}{color.END}\n\n")